Updated how the homography is validated (which should be more robust now). Added new parameters Homography/allCornersVisible, Homography/minAngle and General/autoPauseOnDetection.

git-svn-id: http://find-object.googlecode.com/svn/trunk/find_object@377 620bd6b2-0a58-f614-fd9a-1bd335dccda9

include/find_object/DetectionInfo.h

@@ -18,7 +18,24 @@
 class DetectionInfo
 {
 public:
-	enum TimeStamp{kTimeKeypointDetection, kTimeDescriptorExtraction, kTimeIndexing, kTimeMatching, kTimeHomography, kTimeTotal};
+	enum TimeStamp{
+		kTimeKeypointDetection,
+		kTimeDescriptorExtraction,
+		kTimeIndexing,
+		kTimeMatching,
+		kTimeHomography,
+		kTimeTotal
+	};
+	enum RejectedCode{
+		kRejectedUndef,
+		kRejectedLowMatches,
+		kRejectedLowInliers,
+		kRejectedSuperposed,
+		kRejectedAllInliers,
+		kRejectedNotValid,
+		kRejectedCornersOutside,
+		kRejectedByAngle
+	};
 
 public:
 	DetectionInfo() :
@@ -27,6 +44,7 @@ public:
 	{}
 
 public:
+	// Those maps have the same size
 	QMultiMap<int, QTransform> objDetected_;
 	QMultiMap<int, QSize> objDetectedSizes_; // Object ID <width, height> match the number of detected objects
 	QMultiMap<int, QString > objDetectedFilenames_; // Object ID <filename> match the number of detected objects
@@ -40,8 +58,12 @@ public:
 	cv::Mat sceneDescriptors_;
 	QMultiMap<int, int> sceneWords_;
 	QMap<int, QMultiMap<int, int> > matches_; // ObjectID Map< ObjectDescriptorIndex, SceneDescriptorIndex >, match the number of objects
+
+	// Those maps have the same size
 	QMultiMap<int, QMultiMap<int, int> > rejectedInliers_; // ObjectID Map< ObjectDescriptorIndex, SceneDescriptorIndex >
 	QMultiMap<int, QMultiMap<int, int> > rejectedOutliers_; // ObjectID Map< ObjectDescriptorIndex, SceneDescriptorIndex >
+	QMultiMap<int, RejectedCode> rejectedCodes_; // ObjectID rejected code
+
 	float minMatchedDistance_;
 	float maxMatchedDistance_;
 };

include/find_object/Settings.h

@@ -193,6 +193,7 @@ class FINDOBJECT_EXP Settings
 	PARAMETER(General, vocabularyIncremental, bool, false, "The vocabulary is created incrementally. When new objects are added, their descriptors are compared to those already in vocabulary to find if the visual word already exist or not. \"NearestNeighbor/nndrRatio\" is used to compare descriptors.");
 	PARAMETER(General, vocabularyUpdateMinWords, int, 2000, "When the vocabulary is incremental (see \"General/vocabularyIncremental\"), after X words added to vocabulary, the internal index is updated with new words. This parameter lets avoiding to reconstruct the whole nearest neighbor index after each time descriptors of an object are added to vocabulary. 0 means no incremental update.");
 	PARAMETER(General, sendNoObjDetectedEvents, bool, false, "When there are no objects detected, send an empty object detection event.");
+	PARAMETER(General, autoPauseOnDetection, bool, false, "Auto pause the camera when an object is detected.");
 
 	PARAMETER(Homography, homographyComputed, bool, true, "Compute homography? On ROS, this is required to publish objects detected.");
 	PARAMETER(Homography, method, QString, "1:LMEDS;RANSAC", "Type of the robust estimation algorithm: least-median algorithm or RANSAC algorithm.");
@@ -200,6 +201,8 @@ class FINDOBJECT_EXP Settings
 	PARAMETER(Homography, minimumInliers, int, 10, "Minimum inliers to accept the homography. Value must be >= 4.");
 	PARAMETER(Homography, ignoreWhenAllInliers, bool, false, "Ignore homography when all features are inliers (sometimes when the homography doesn't converge, it returns the best homography with all features as inliers).");
 	PARAMETER(Homography, rectBorderWidth, int, 4, "Homography rectangle border width.");
+	PARAMETER(Homography, allCornersVisible, bool, false, "All corners of the detected object must be visible in the scene.");
+	PARAMETER(Homography, minAngle, int, 0, "(Degrees) Homography minimum angle. Set 0 to disable. When the angle is very small, this is a good indication that the homography is wrong. A good value is over 60 degrees.");
 
 public:
 	virtual ~Settings(){}

src/FindObject.cpp

@@ -17,6 +17,7 @@
 #include <QtCore/QFileInfo>
 #include <QtCore/QStringList>
 #include <QtCore/QTime>
+#include <QtGui/QGraphicsRectItem>
 #include <stdio.h>
 
 FindObject::FindObject(QObject * parent) :
@@ -510,7 +511,8 @@ public:
 				matches_(matches),
 				objectId_(objectId),
 				kptsA_(kptsA),
-				kptsB_(kptsB)
+				kptsB_(kptsB),
+				code_(DetectionInfo::kRejectedUndef)
 	{
 		Q_ASSERT(matches && kptsA && kptsB);
 	}
@@ -523,6 +525,7 @@ public:
 	QMultiMap<int, int> getInliers() const {return inliers_;}
 	QMultiMap<int, int> getOutliers() const {return outliers_;}
 	const cv::Mat & getHomography() const {return h_;}
+	DetectionInfo::RejectedCode rejectedCode() const {return code_;}
 
 protected:
 	virtual void run()
@@ -565,15 +568,20 @@ protected:
 				}
 			}
 
-			// ignore homography when all features are inliers
 			if(inliers_.size() == (int)outlierMask_.size() && !h_.empty())
 			{
 				if(Settings::getHomography_ignoreWhenAllInliers() || cv::countNonZero(h_) < 1)
 				{
+					// ignore homography when all features are inliers
 					h_ = cv::Mat();
+					code_ = DetectionInfo::kRejectedAllInliers;
 				}
 			}
 		}
+		else
+		{
+			code_ = DetectionInfo::kRejectedLowMatches;
+		}
 
 		//UINFO("Homography Object %d time=%d ms", objectIndex_, time.elapsed());
 	}
@@ -582,6 +590,7 @@ private:
 	int objectId_;
 	const std::vector<cv::KeyPoint> * kptsA_;
 	const std::vector<cv::KeyPoint> * kptsB_;
+	DetectionInfo::RejectedCode code_;
 
 	std::vector<int> indexesA_;
 	std::vector<int> indexesB_;
@@ -860,65 +869,129 @@ bool FindObject::detect(const cv::Mat & image, DetectionInfo & info)
 						threads[j]->wait();
 
 						int id = threads[j]->getObjectId();
-
+						QTransform hTransform;
-						if(!threads[j]->getHomography().empty())
+						DetectionInfo::RejectedCode code = DetectionInfo::kRejectedUndef;
+						if(threads[j]->getHomography().empty())
 						{
-							if(threads[j]->getInliers().size() >= Settings::getHomography_minimumInliers())
+							code = threads[j]->rejectedCode();
+						}
+						if(code == DetectionInfo::kRejectedUndef &&
+						   threads[j]->getInliers().size() < Settings::getHomography_minimumInliers()	)
+						{
+							code = DetectionInfo::kRejectedLowInliers;
+						}
+						if(code == DetectionInfo::kRejectedUndef)
+						{
+							const cv::Mat & H = threads[j]->getHomography();
+							hTransform = QTransform(
+								H.at<double>(0,0), H.at<double>(1,0), H.at<double>(2,0),
+								H.at<double>(0,1), H.at<double>(1,1), H.at<double>(2,1),
+								H.at<double>(0,2), H.at<double>(1,2), H.at<double>(2,2));
+
+							// is homography valid?
+							// Here we use mapToScene() from QGraphicsItem instead
+							// of QTransform::map() because if the homography is not valid,
+							// huge errors are set by the QGraphicsItem and not by QTransform::map();
+							QRectF objectRect = objects_.value(id)->rect();
+							QGraphicsRectItem item(objectRect);
+							item.setTransform(hTransform);
+							QPolygonF rectH = item.mapToScene(item.rect());
+
+							// If a point is outside of 2x times the surface of the scene, homography is invalid.
+							for(int p=0; p<rectH.size(); ++p)
 							{
-								const cv::Mat & H = threads[j]->getHomography();
+								if(rectH.at(p).x() < -image.cols || rectH.at(p).x() > image.cols*2 ||
-								QTransform hTransform(
+								   rectH.at(p).y() < -image.rows || rectH.at(p).y() > image.rows*2)
-									H.at<double>(0,0), H.at<double>(1,0), H.at<double>(2,0),
-									H.at<double>(0,1), H.at<double>(1,1), H.at<double>(2,1),
-									H.at<double>(0,2), H.at<double>(1,2), H.at<double>(2,2));
-
-								int distance = Settings::getGeneral_multiDetectionRadius(); // in pixels
-								if(Settings::getGeneral_multiDetection())
 								{
-									// Get the outliers and recompute homography with them
+									code= DetectionInfo::kRejectedNotValid;
-									matchesList.push_back(threads[j]->getOutliers());
+									break;
-									matchesId.push_back(id);
+								}
+							}
 
-									// compute distance from previous added same objects...
+							// angle
-									QMultiMap<int, QTransform>::iterator objIter = info.objDetected_.find(id);
+							if(code == DetectionInfo::kRejectedUndef &&
-									for(;objIter!=info.objDetected_.end() && objIter.key() == id; ++objIter)
+							   Settings::getHomography_minAngle() > 0)
+							{
+								for(int a=0; a<rectH.size(); ++a)
+								{
+									//  Find the smaller angle
+									QLineF ab(rectH.at(a).x(), rectH.at(a).y(), rectH.at((a+1)%4).x(), rectH.at((a+1)%4).y());
+									QLineF cb(rectH.at((a+1)%4).x(), rectH.at((a+1)%4).y(), rectH.at((a+2)%4).x(), rectH.at((a+2)%4).y());
+									float angle =  ab.angle(cb);
+									float minAngle = (float)Settings::getHomography_minAngle();
+									if(angle < minAngle ||
+									   angle > 180.0-minAngle)
 									{
-										qreal dx = objIter.value().m31() - hTransform.m31();
+										code = DetectionInfo::kRejectedByAngle;
-										qreal dy = objIter.value().m32() - hTransform.m32();
+										break;
-										int d = (int)sqrt(dx*dx + dy*dy);
+									}
-										if(d < distance)
+								}
-										{
+							}
-											distance = d;
+
-										}
+							// multi detection
+							if(code == DetectionInfo::kRejectedUndef &&
+							   Settings::getGeneral_multiDetection())
+							{
+								int distance = Settings::getGeneral_multiDetectionRadius(); // in pixels
+								// Get the outliers and recompute homography with them
+								matchesList.push_back(threads[j]->getOutliers());
+								matchesId.push_back(id);
+
+								// compute distance from previous added same objects...
+								QMultiMap<int, QTransform>::iterator objIter = info.objDetected_.find(id);
+								for(;objIter!=info.objDetected_.end() && objIter.key() == id; ++objIter)
+								{
+									qreal dx = objIter.value().m31() - hTransform.m31();
+									qreal dy = objIter.value().m32() - hTransform.m32();
+									int d = (int)sqrt(dx*dx + dy*dy);
+									if(d < distance)
+									{
+										distance = d;
 									}
 								}
 
-								if(distance >= Settings::getGeneral_multiDetectionRadius())
+								if(distance < Settings::getGeneral_multiDetectionRadius())
 								{
-									QRect rect = objects_.value(id)->rect();
+									code = DetectionInfo::kRejectedSuperposed;
-									info.objDetected_.insert(id, hTransform);
-									info.objDetectedSizes_.insert(id, rect.size());
-									info.objDetectedInliers_.insert(id, threads[j]->getInliers());
-									info.objDetectedOutliers_.insert(id, threads[j]->getOutliers());
-									info.objDetectedInliersCount_.insert(id, threads[j]->getInliers().size());
-									info.objDetectedOutliersCount_.insert(id, threads[j]->getOutliers().size());
-									info.objDetectedFilenames_.insert(id, objects_.value(id)->filename());
-								}
-								else
-								{
-									info.rejectedInliers_.insert(id, threads[j]->getInliers());
-									info.rejectedOutliers_.insert(id, threads[j]->getOutliers());
 								}
 							}
-							else
+
+							// Corners visible
+							if(code == DetectionInfo::kRejectedUndef &&
+							   Settings::getHomography_allCornersVisible())
 							{
-								info.rejectedInliers_.insert(id, threads[j]->getInliers());
+								// Now verify if all corners are in the scene
-								info.rejectedOutliers_.insert(id, threads[j]->getOutliers());
+								QRectF sceneRect(0,0,image.cols, image.rows);
+								for(int p=0; p<rectH.size(); ++p)
+								{
+									if(!sceneRect.contains(QPointF(rectH.at(p).x(), rectH.at(p).y())))
+									{
+										code = DetectionInfo::kRejectedCornersOutside;
+										break;
+									}
+								}
 							}
 						}
+
+						if(code == DetectionInfo::kRejectedUndef)
+						{
+							// Accepted!
+							//std::cout << "H= " << threads[j]->getHomography() << std::endl;
+
+							info.objDetected_.insert(id, hTransform);
+							info.objDetectedSizes_.insert(id, objects_.value(id)->rect().size());
+							info.objDetectedInliers_.insert(id, threads[j]->getInliers());
+							info.objDetectedOutliers_.insert(id, threads[j]->getOutliers());
+							info.objDetectedInliersCount_.insert(id, threads[j]->getInliers().size());
+							info.objDetectedOutliersCount_.insert(id, threads[j]->getOutliers().size());
+							info.objDetectedFilenames_.insert(id, objects_.value(id)->filename());
+						}
 						else
 						{
+							//Rejected!
 							info.rejectedInliers_.insert(id, threads[j]->getInliers());
 							info.rejectedOutliers_.insert(id, threads[j]->getOutliers());
+							info.rejectedCodes_.insert(id, code);
 						}
 					}
 				}

src/MainWindow.cpp

@@ -32,6 +32,7 @@
 #include <QtCore/QFile>
 #include <QtCore/QBuffer>
 #include <QtCore/QThread>
+#include <QtCore/QLineF>
 
 #include <QtGui/QFileDialog>
 #include <QtGui/QMessageBox>
@@ -129,7 +130,7 @@ MainWindow::MainWindow(FindObject * findObject, Camera * camera, QWidget * paren
 	ui_->menuView->addAction(ui_->dockWidget_plot->toggleViewAction());
 	connect(ui_->toolBox, SIGNAL(parametersChanged(const QStringList &)), this, SLOT(notifyParametersChanged(const QStringList &)));
 
-	ui_->imageView_source->setGraphicsViewMode(false);
+	ui_->imageView_source->setGraphicsViewMode(true);
 	ui_->imageView_source->setTextLabel(tr("Press \"space\" to start the camera..."));
 	ui_->imageView_source->setMirrorView(Settings::getGeneral_mirrorView());
 	connect((QCheckBox*)ui_->toolBox->getParameterWidget(Settings::kGeneral_mirrorView()),
@@ -963,22 +964,39 @@ void MainWindow::update(const cv::Mat & image)
 				QLabel * label = ui_->dockWidget_objects->findChild<QLabel*>(QString("%1detection").arg(id));
 				QMultiMap<int, int> rejectedInliers = info.rejectedInliers_.value(id);
 				QMultiMap<int, int> rejectedOutliers = info.rejectedOutliers_.value(id);
-				if(jter.value().size() < Settings::getHomography_minimumInliers())
+				int rejectedCode = info.rejectedCodes_.value(id);
+				if(rejectedCode == DetectionInfo::kRejectedLowMatches)
 				{
 					label->setText(QString("Too low matches (%1)").arg(jter.value().size()));
 				}
-				else if(rejectedInliers.size() >= Settings::getHomography_minimumInliers())
+				else if(rejectedCode == DetectionInfo::kRejectedAllInliers)
 				{
 					label->setText(QString("Ignored, all inliers (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size()));
 				}
-				else
+				else if(rejectedCode == DetectionInfo::kRejectedNotValid)
+				{
+					label->setText(QString("Not valid homography (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size()));
+				}
+				else if(rejectedCode == DetectionInfo::kRejectedLowInliers)
 				{
 					label->setText(QString("Too low inliers (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size()));
 				}
-
+				else if(rejectedCode == DetectionInfo::kRejectedCornersOutside)
+				{
+					label->setText(QString("Corners not visible (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size()));
+				}
+				else if(rejectedCode == DetectionInfo::kRejectedByAngle)
+				{
+					label->setText(QString("Angle too small (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size()));
+				}
 			}
 		}
 
+		if(camera_->isRunning() && Settings::getGeneral_autoPauseOnDetection() && info.objDetected_.size())
+		{
+			this->pauseProcessing();
+		}
+
 		// Add homography rectangles when homographies are computed
 		QMultiMap<int, QMultiMap<int,int> >::const_iterator inliersIter = info.objDetectedInliers_.constBegin();
 		QMultiMap<int, QMultiMap<int,int> >::const_iterator outliersIter = info.objDetectedOutliers_.constBegin();

src/RectItem.cpp

@@ -48,7 +48,35 @@ void RectItem::showDescription()
 			placeHolder_->setBrush(QBrush(QColor ( 0, 0, 0, 170 ))); // Black transparent background
 			QGraphicsTextItem * text = new QGraphicsTextItem(placeHolder_);
 			text->setDefaultTextColor(this->pen().color().rgb());
-			text->setPlainText(tr("Object=%1").arg(id_));
+			QTransform t = this->transform();
+			QPolygonF rectH = this->mapToScene(this->rect());
+			float angle = 90.0f;
+			for(int a=0; a<rectH.size(); ++a)
+			{
+				//  Find the smaller angle
+				QLineF ab(rectH.at(a).x(), rectH.at(a).y(), rectH.at((a+1)%4).x(), rectH.at((a+1)%4).y());
+				QLineF cb(rectH.at((a+1)%4).x(), rectH.at((a+1)%4).y(), rectH.at((a+2)%4).x(), rectH.at((a+2)%4).y());
+				float angleTmp =  ab.angle(cb);
+				if(angleTmp > 90.0f)
+				{
+					angleTmp  = 180.0f - angleTmp;
+				}
+				if(angleTmp < angle)
+				{
+					angle = angleTmp;
+				}
+			}
+			text->setPlainText(tr(
+					"Object=%1\n"
+					"Homography= [\n"
+					"            %2 %3 %4\n"
+					"            %5 %6 %7\n"
+					"            %8 %9 %10]\n"
+					"Angle=%11").arg(id_)
+					.arg(t.m11()).arg(t.m12()).arg(t.m13())
+					.arg(t.m21()).arg(t.m22()).arg(t.m23())
+					.arg(t.m31()).arg(t.m32()).arg(t.m33())
+					.arg(angle));
 			placeHolder_->setRect(text->boundingRect());
 		}