diff --git a/include/find_object/DetectionInfo.h b/include/find_object/DetectionInfo.h index eb5c6fb3..40a1a2fe 100644 --- a/include/find_object/DetectionInfo.h +++ b/include/find_object/DetectionInfo.h @@ -18,7 +18,24 @@ class DetectionInfo { public: - enum TimeStamp{kTimeKeypointDetection, kTimeDescriptorExtraction, kTimeIndexing, kTimeMatching, kTimeHomography, kTimeTotal}; + enum TimeStamp{ + kTimeKeypointDetection, + kTimeDescriptorExtraction, + kTimeIndexing, + kTimeMatching, + kTimeHomography, + kTimeTotal + }; + enum RejectedCode{ + kRejectedUndef, + kRejectedLowMatches, + kRejectedLowInliers, + kRejectedSuperposed, + kRejectedAllInliers, + kRejectedNotValid, + kRejectedCornersOutside, + kRejectedByAngle + }; public: DetectionInfo() : @@ -27,6 +44,7 @@ public: {} public: + // Those maps have the same size QMultiMap objDetected_; QMultiMap objDetectedSizes_; // Object ID match the number of detected objects QMultiMap objDetectedFilenames_; // Object ID match the number of detected objects @@ -40,8 +58,12 @@ public: cv::Mat sceneDescriptors_; QMultiMap sceneWords_; QMap > matches_; // ObjectID Map< ObjectDescriptorIndex, SceneDescriptorIndex >, match the number of objects + + // Those maps have the same size QMultiMap > rejectedInliers_; // ObjectID Map< ObjectDescriptorIndex, SceneDescriptorIndex > QMultiMap > rejectedOutliers_; // ObjectID Map< ObjectDescriptorIndex, SceneDescriptorIndex > + QMultiMap rejectedCodes_; // ObjectID rejected code + float minMatchedDistance_; float maxMatchedDistance_; }; diff --git a/include/find_object/Settings.h b/include/find_object/Settings.h index 81da5e85..a0f9a466 100644 --- a/include/find_object/Settings.h +++ b/include/find_object/Settings.h @@ -193,6 +193,7 @@ class FINDOBJECT_EXP Settings PARAMETER(General, vocabularyIncremental, bool, false, "The vocabulary is created incrementally. When new objects are added, their descriptors are compared to those already in vocabulary to find if the visual word already exist or not. \"NearestNeighbor/nndrRatio\" is used to compare descriptors."); PARAMETER(General, vocabularyUpdateMinWords, int, 2000, "When the vocabulary is incremental (see \"General/vocabularyIncremental\"), after X words added to vocabulary, the internal index is updated with new words. This parameter lets avoiding to reconstruct the whole nearest neighbor index after each time descriptors of an object are added to vocabulary. 0 means no incremental update."); PARAMETER(General, sendNoObjDetectedEvents, bool, false, "When there are no objects detected, send an empty object detection event."); + PARAMETER(General, autoPauseOnDetection, bool, false, "Auto pause the camera when an object is detected."); PARAMETER(Homography, homographyComputed, bool, true, "Compute homography? On ROS, this is required to publish objects detected."); PARAMETER(Homography, method, QString, "1:LMEDS;RANSAC", "Type of the robust estimation algorithm: least-median algorithm or RANSAC algorithm."); @@ -200,6 +201,8 @@ class FINDOBJECT_EXP Settings PARAMETER(Homography, minimumInliers, int, 10, "Minimum inliers to accept the homography. Value must be >= 4."); PARAMETER(Homography, ignoreWhenAllInliers, bool, false, "Ignore homography when all features are inliers (sometimes when the homography doesn't converge, it returns the best homography with all features as inliers)."); PARAMETER(Homography, rectBorderWidth, int, 4, "Homography rectangle border width."); + PARAMETER(Homography, allCornersVisible, bool, false, "All corners of the detected object must be visible in the scene."); + PARAMETER(Homography, minAngle, int, 0, "(Degrees) Homography minimum angle. Set 0 to disable. When the angle is very small, this is a good indication that the homography is wrong. A good value is over 60 degrees."); public: virtual ~Settings(){} diff --git a/src/FindObject.cpp b/src/FindObject.cpp index f6926d32..f756df7c 100644 --- a/src/FindObject.cpp +++ b/src/FindObject.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include FindObject::FindObject(QObject * parent) : @@ -510,7 +511,8 @@ public: matches_(matches), objectId_(objectId), kptsA_(kptsA), - kptsB_(kptsB) + kptsB_(kptsB), + code_(DetectionInfo::kRejectedUndef) { Q_ASSERT(matches && kptsA && kptsB); } @@ -523,6 +525,7 @@ public: QMultiMap getInliers() const {return inliers_;} QMultiMap getOutliers() const {return outliers_;} const cv::Mat & getHomography() const {return h_;} + DetectionInfo::RejectedCode rejectedCode() const {return code_;} protected: virtual void run() @@ -565,15 +568,20 @@ protected: } } - // ignore homography when all features are inliers if(inliers_.size() == (int)outlierMask_.size() && !h_.empty()) { if(Settings::getHomography_ignoreWhenAllInliers() || cv::countNonZero(h_) < 1) { + // ignore homography when all features are inliers h_ = cv::Mat(); + code_ = DetectionInfo::kRejectedAllInliers; } } } + else + { + code_ = DetectionInfo::kRejectedLowMatches; + } //UINFO("Homography Object %d time=%d ms", objectIndex_, time.elapsed()); } @@ -582,6 +590,7 @@ private: int objectId_; const std::vector * kptsA_; const std::vector * kptsB_; + DetectionInfo::RejectedCode code_; std::vector indexesA_; std::vector indexesB_; @@ -860,65 +869,129 @@ bool FindObject::detect(const cv::Mat & image, DetectionInfo & info) threads[j]->wait(); int id = threads[j]->getObjectId(); - - if(!threads[j]->getHomography().empty()) + QTransform hTransform; + DetectionInfo::RejectedCode code = DetectionInfo::kRejectedUndef; + if(threads[j]->getHomography().empty()) { - if(threads[j]->getInliers().size() >= Settings::getHomography_minimumInliers()) + code = threads[j]->rejectedCode(); + } + if(code == DetectionInfo::kRejectedUndef && + threads[j]->getInliers().size() < Settings::getHomography_minimumInliers() ) + { + code = DetectionInfo::kRejectedLowInliers; + } + if(code == DetectionInfo::kRejectedUndef) + { + const cv::Mat & H = threads[j]->getHomography(); + hTransform = QTransform( + H.at(0,0), H.at(1,0), H.at(2,0), + H.at(0,1), H.at(1,1), H.at(2,1), + H.at(0,2), H.at(1,2), H.at(2,2)); + + // is homography valid? + // Here we use mapToScene() from QGraphicsItem instead + // of QTransform::map() because if the homography is not valid, + // huge errors are set by the QGraphicsItem and not by QTransform::map(); + QRectF objectRect = objects_.value(id)->rect(); + QGraphicsRectItem item(objectRect); + item.setTransform(hTransform); + QPolygonF rectH = item.mapToScene(item.rect()); + + // If a point is outside of 2x times the surface of the scene, homography is invalid. + for(int p=0; pgetHomography(); - QTransform hTransform( - H.at(0,0), H.at(1,0), H.at(2,0), - H.at(0,1), H.at(1,1), H.at(2,1), - H.at(0,2), H.at(1,2), H.at(2,2)); - - int distance = Settings::getGeneral_multiDetectionRadius(); // in pixels - if(Settings::getGeneral_multiDetection()) + if(rectH.at(p).x() < -image.cols || rectH.at(p).x() > image.cols*2 || + rectH.at(p).y() < -image.rows || rectH.at(p).y() > image.rows*2) { - // Get the outliers and recompute homography with them - matchesList.push_back(threads[j]->getOutliers()); - matchesId.push_back(id); + code= DetectionInfo::kRejectedNotValid; + break; + } + } - // compute distance from previous added same objects... - QMultiMap::iterator objIter = info.objDetected_.find(id); - for(;objIter!=info.objDetected_.end() && objIter.key() == id; ++objIter) + // angle + if(code == DetectionInfo::kRejectedUndef && + Settings::getHomography_minAngle() > 0) + { + for(int a=0; a 180.0-minAngle) { - qreal dx = objIter.value().m31() - hTransform.m31(); - qreal dy = objIter.value().m32() - hTransform.m32(); - int d = (int)sqrt(dx*dx + dy*dy); - if(d < distance) - { - distance = d; - } + code = DetectionInfo::kRejectedByAngle; + break; + } + } + } + + // multi detection + if(code == DetectionInfo::kRejectedUndef && + Settings::getGeneral_multiDetection()) + { + int distance = Settings::getGeneral_multiDetectionRadius(); // in pixels + // Get the outliers and recompute homography with them + matchesList.push_back(threads[j]->getOutliers()); + matchesId.push_back(id); + + // compute distance from previous added same objects... + QMultiMap::iterator objIter = info.objDetected_.find(id); + for(;objIter!=info.objDetected_.end() && objIter.key() == id; ++objIter) + { + qreal dx = objIter.value().m31() - hTransform.m31(); + qreal dy = objIter.value().m32() - hTransform.m32(); + int d = (int)sqrt(dx*dx + dy*dy); + if(d < distance) + { + distance = d; } } - if(distance >= Settings::getGeneral_multiDetectionRadius()) + if(distance < Settings::getGeneral_multiDetectionRadius()) { - QRect rect = objects_.value(id)->rect(); - info.objDetected_.insert(id, hTransform); - info.objDetectedSizes_.insert(id, rect.size()); - info.objDetectedInliers_.insert(id, threads[j]->getInliers()); - info.objDetectedOutliers_.insert(id, threads[j]->getOutliers()); - info.objDetectedInliersCount_.insert(id, threads[j]->getInliers().size()); - info.objDetectedOutliersCount_.insert(id, threads[j]->getOutliers().size()); - info.objDetectedFilenames_.insert(id, objects_.value(id)->filename()); - } - else - { - info.rejectedInliers_.insert(id, threads[j]->getInliers()); - info.rejectedOutliers_.insert(id, threads[j]->getOutliers()); + code = DetectionInfo::kRejectedSuperposed; } } - else + + // Corners visible + if(code == DetectionInfo::kRejectedUndef && + Settings::getHomography_allCornersVisible()) { - info.rejectedInliers_.insert(id, threads[j]->getInliers()); - info.rejectedOutliers_.insert(id, threads[j]->getOutliers()); + // Now verify if all corners are in the scene + QRectF sceneRect(0,0,image.cols, image.rows); + for(int p=0; pimageView_source->setMirrorView(Settings::getGeneral_mirrorView()); connect((QCheckBox*)ui_->toolBox->getParameterWidget(Settings::kGeneral_mirrorView()), @@ -963,22 +964,39 @@ void MainWindow::update(const cv::Mat & image) QLabel * label = ui_->dockWidget_objects->findChild(QString("%1detection").arg(id)); QMultiMap rejectedInliers = info.rejectedInliers_.value(id); QMultiMap rejectedOutliers = info.rejectedOutliers_.value(id); - if(jter.value().size() < Settings::getHomography_minimumInliers()) + int rejectedCode = info.rejectedCodes_.value(id); + if(rejectedCode == DetectionInfo::kRejectedLowMatches) { label->setText(QString("Too low matches (%1)").arg(jter.value().size())); } - else if(rejectedInliers.size() >= Settings::getHomography_minimumInliers()) + else if(rejectedCode == DetectionInfo::kRejectedAllInliers) { label->setText(QString("Ignored, all inliers (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size())); } - else + else if(rejectedCode == DetectionInfo::kRejectedNotValid) + { + label->setText(QString("Not valid homography (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size())); + } + else if(rejectedCode == DetectionInfo::kRejectedLowInliers) { label->setText(QString("Too low inliers (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size())); } - + else if(rejectedCode == DetectionInfo::kRejectedCornersOutside) + { + label->setText(QString("Corners not visible (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size())); + } + else if(rejectedCode == DetectionInfo::kRejectedByAngle) + { + label->setText(QString("Angle too small (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size())); + } } } + if(camera_->isRunning() && Settings::getGeneral_autoPauseOnDetection() && info.objDetected_.size()) + { + this->pauseProcessing(); + } + // Add homography rectangles when homographies are computed QMultiMap >::const_iterator inliersIter = info.objDetectedInliers_.constBegin(); QMultiMap >::const_iterator outliersIter = info.objDetectedOutliers_.constBegin(); diff --git a/src/RectItem.cpp b/src/RectItem.cpp index e9dd1a65..10d268e6 100644 --- a/src/RectItem.cpp +++ b/src/RectItem.cpp @@ -48,7 +48,35 @@ void RectItem::showDescription() placeHolder_->setBrush(QBrush(QColor ( 0, 0, 0, 170 ))); // Black transparent background QGraphicsTextItem * text = new QGraphicsTextItem(placeHolder_); text->setDefaultTextColor(this->pen().color().rgb()); - text->setPlainText(tr("Object=%1").arg(id_)); + QTransform t = this->transform(); + QPolygonF rectH = this->mapToScene(this->rect()); + float angle = 90.0f; + for(int a=0; a 90.0f) + { + angleTmp = 180.0f - angleTmp; + } + if(angleTmp < angle) + { + angle = angleTmp; + } + } + text->setPlainText(tr( + "Object=%1\n" + "Homography= [\n" + " %2 %3 %4\n" + " %5 %6 %7\n" + " %8 %9 %10]\n" + "Angle=%11").arg(id_) + .arg(t.m11()).arg(t.m12()).arg(t.m13()) + .arg(t.m21()).arg(t.m22()).arg(t.m23()) + .arg(t.m31()).arg(t.m32()).arg(t.m33()) + .arg(angle)); placeHolder_->setRect(text->boundingRect()); }