diff --git a/include/find_object/DetectionInfo.h b/include/find_object/DetectionInfo.h index 915604d4..c11e7853 100644 --- a/include/find_object/DetectionInfo.h +++ b/include/find_object/DetectionInfo.h @@ -42,6 +42,7 @@ public: enum TimeStamp{ kTimeKeypointDetection, kTimeDescriptorExtraction, + kTimeSubPixelRefining, kTimeSkewAffine, kTimeIndexing, kTimeMatching, diff --git a/include/find_object/Settings.h b/include/find_object/Settings.h index 04da5c2e..7a6e78ad 100644 --- a/include/find_object/Settings.h +++ b/include/find_object/Settings.h @@ -120,6 +120,10 @@ class FINDOBJECT_EXP Settings PARAMETER(Feature2D, 3MaxFeatures, int, 0, "Maximum features per image. If the number of features extracted is over this threshold, only X features with the highest response are kept. 0 means all features are kept."); PARAMETER(Feature2D, 4Affine, bool, false, "(ASIFT) Extract features on multiple affine transformations of the image."); PARAMETER(Feature2D, 5AffineCount, int, 6, "(ASIFT) Higher the value, more affine transformations will be done."); + PARAMETER(Feature2D, 6SubPix, bool, FINDOBJECT_NONFREE != 1, "Refines the corner locations. With SIFT/SURF, features are already subpixel, so no need to activate this."); + PARAMETER(Feature2D, 7SubPixWinSize, int, 3, "Half of the side length of the search window. For example, if winSize=Size(5,5) , then a 5*2+1 x 5*2+1 = 11 x 11 search window is used."); + PARAMETER(Feature2D, 8SubPixIterations, int, 30, "The process of corner position refinement stops after X iterations."); + PARAMETER(Feature2D, 9SubPixEps, float, 0.02, "The process of corner position refinement stops when the corner position moves by less than epsilon on some iteration."); PARAMETER(Feature2D, Brief_bytes, int, 32, "Bytes is a length of descriptor in bytes. It can be equal 16, 32 or 64 bytes."); @@ -255,6 +259,11 @@ class FINDOBJECT_EXP Settings PARAMETER(Homography, rectBorderWidth, int, 4, "Homography rectangle border width."); PARAMETER(Homography, allCornersVisible, bool, false, "All corners of the detected object must be visible in the scene."); PARAMETER(Homography, minAngle, int, 0, "(Degrees) Homography minimum angle. Set 0 to disable. When the angle is very small, this is a good indication that the homography is wrong. A good value is over 60 degrees."); + PARAMETER(Homography, opticalFlow, bool, false, "Activate optical flow to refine matched features before computing the homography."); + PARAMETER(Homography, opticalFlowWinSize, int, 16, "Size of the search window at each pyramid level."); + PARAMETER(Homography, opticalFlowMaxLevel, int, 3, "0-based maximal pyramid level number; if set to 0, pyramids are not used (single level), if set to 1, two levels are used, and so on; if pyramids are passed to input then algorithm will use as many levels as pyramids have but no more than maxLevel."); + PARAMETER(Homography, opticalFlowIterations, int, 30, "Specifying the termination criteria of the iterative search algorithm (after the specified maximum number of iterations)."); + PARAMETER(Homography, opticalFlowEps, float, 0.01, "Specifying the termination criteria of the iterative search algorithm (when the search window moves by less than epsilon)."); public: virtual ~Settings(){} diff --git a/src/FindObject.cpp b/src/FindObject.cpp index 81d211cd..77d1e40a 100644 --- a/src/FindObject.cpp +++ b/src/FindObject.cpp @@ -377,7 +377,8 @@ public: phi_(phi), timeSkewAffine_(0), timeDetection_(0), - timeExtraction_(0) + timeExtraction_(0), + timeSubPix_(0) { UASSERT(detector && extractor); } @@ -388,6 +389,7 @@ public: int timeSkewAffine() const {return timeSkewAffine_;} int timeDetection() const {return timeDetection_;} int timeExtraction() const {return timeExtraction_;} + int timeSubPix() const {return timeSubPix_;} protected: virtual void run() @@ -422,6 +424,24 @@ protected: keypoints_[i].pt.x = pa.at(0,0); keypoints_[i].pt.y = pa.at(1,0); } + + if(keypoints_.size() && Settings::getFeature2D_6SubPix()) + { + // Sub pixel should be done after descriptors extraction + std::vector corners; + cv::KeyPoint::convert(keypoints_, corners); + cv::cornerSubPix(image_, + corners, + cv::Size(Settings::getFeature2D_7SubPixWinSize(), Settings::getFeature2D_7SubPixWinSize()), + cv::Size(-1,-1), + cv::TermCriteria( CV_TERMCRIT_EPS + CV_TERMCRIT_ITER, Settings::getFeature2D_8SubPixIterations(), Settings::getFeature2D_9SubPixEps() )); + UASSERT(corners.size() == keypoints_.size()); + for(unsigned int i=0; i corners; + cv::KeyPoint::convert(keypoints_, corners); + cv::cornerSubPix(image_, + corners, + cv::Size(Settings::getFeature2D_7SubPixWinSize(), Settings::getFeature2D_7SubPixWinSize()), + cv::Size(-1,-1), + cv::TermCriteria( CV_TERMCRIT_EPS + CV_TERMCRIT_ITER, Settings::getFeature2D_8SubPixIterations(), Settings::getFeature2D_9SubPixEps() )); + UASSERT(corners.size() == keypoints_.size()); + for(unsigned int i=0; itimeSkewAffine(); timeDetection_ += threads[k]->timeDetection(); timeExtraction_ += threads[k]->timeExtraction(); + timeSubPix_ += threads[k]->timeSubPix(); } } } @@ -583,6 +624,7 @@ private: int timeSkewAffine_; int timeDetection_; int timeExtraction_; + int timeSubPix_; }; void FindObject::updateObjects(const QList & ids) @@ -876,11 +918,15 @@ public: const QMultiMap * matches, // int objectId, const std::vector * kptsA, - const std::vector * kptsB) : + const std::vector * kptsB, + const cv::Mat & imageA, // image only required if opticalFlow is on + const cv::Mat & imageB) : // image only required if opticalFlow is on matches_(matches), objectId_(objectId), kptsA_(kptsA), kptsB_(kptsB), + imageA_(imageA), + imageB_(imageB), code_(DetectionInfo::kRejectedUndef) { UASSERT(matches && kptsA && kptsB); @@ -919,6 +965,41 @@ protected: if((int)mpts_1.size() >= Settings::getHomography_minimumInliers()) { + if(Settings::getHomography_opticalFlow()) + { + UASSERT(!imageA_.empty() && !imageB_.empty()); + + cv::Mat imageA = imageA_; + cv::Mat imageB = imageB_; + if(imageA_.cols < imageB_.cols && imageA_.rows < imageB_.rows) + { + // padding, optical flow wants images of the same size + imageA = cv::Mat::zeros(imageB_.size(), imageA_.type()); + imageA_.copyTo(imageA(cv::Rect(0,0,imageA_.cols, imageA_.rows))); + } + if(imageA.size() == imageB.size()) + { + //refine matches + std::vector status; + std::vector err; + cv::calcOpticalFlowPyrLK( + imageA, + imageB_, + mpts_1, + mpts_2, + status, + err, + cv::Size(Settings::getHomography_opticalFlowWinSize(), Settings::getHomography_opticalFlowWinSize()), + Settings::getHomography_opticalFlowMaxLevel(), + cv::TermCriteria(cv::TermCriteria::COUNT+cv::TermCriteria::EPS, Settings::getHomography_opticalFlowIterations(), Settings::getHomography_opticalFlowEps()), + cv::OPTFLOW_LK_GET_MIN_EIGENVALS | cv::OPTFLOW_USE_INITIAL_FLOW, 1e-4); + } + else + { + UERROR("Object's image should be less/equal size of the scene image to use Optical Flow."); + } + } + h_ = findHomography(mpts_1, mpts_2, Settings::getHomographyMethod(), @@ -959,6 +1040,8 @@ private: int objectId_; const std::vector * kptsA_; const std::vector * kptsB_; + cv::Mat imageA_; + cv::Mat imageB_; DetectionInfo::RejectedCode code_; std::vector indexesA_; @@ -1033,6 +1116,7 @@ bool FindObject::detect(const cv::Mat & image, find_object::DetectionInfo & info info.sceneDescriptors_ = extractThread.descriptors(); info.timeStamps_.insert(DetectionInfo::kTimeKeypointDetection, extractThread.timeDetection()); info.timeStamps_.insert(DetectionInfo::kTimeDescriptorExtraction, extractThread.timeExtraction()); + info.timeStamps_.insert(DetectionInfo::kTimeSubPixelRefining, extractThread.timeSubPix()); info.timeStamps_.insert(DetectionInfo::kTimeSkewAffine, extractThread.timeSkewAffine()); bool consistentNNData = (vocabulary_->size()!=0 && vocabulary_->wordToObjects().begin().value()!=-1 && Settings::getGeneral_invertedSearch()) || @@ -1063,9 +1147,9 @@ bool FindObject::detect(const cv::Mat & image, find_object::DetectionInfo & info if(!Settings::getGeneral_invertedSearch()) { + vocabulary_->clear(); // CREATE INDEX for the scene UDEBUG("CREATE INDEX FOR THE SCENE"); - vocabulary_->clear(); words = vocabulary_->addWords(info.sceneDescriptors_, -1, Settings::getGeneral_vocabularyIncremental()); if(!Settings::getGeneral_vocabularyIncremental()) { @@ -1235,7 +1319,13 @@ bool FindObject::detect(const cv::Mat & image, find_object::DetectionInfo & info for(int k=i; kkeypoints(), &info.sceneKeypoints_)); + threads.push_back(new HomographyThread( + &matchesList[k], + objectId, + &objects_.value(objectId)->keypoints(), + &info.sceneKeypoints_, + objects_.value(objectId)->image(), + grayscaleImg)); threads.back()->start(); } @@ -1353,8 +1443,6 @@ bool FindObject::detect(const cv::Mat & image, find_object::DetectionInfo & info if(code == DetectionInfo::kRejectedUndef) { // Accepted! - //std::cout << "H= " << threads[j]->getHomography() << std::endl; - info.objDetected_.insert(id, hTransform); info.objDetectedSizes_.insert(id, objects_.value(id)->rect().size()); info.objDetectedInliers_.insert(id, threads[j]->getInliers()); diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index 52e6475b..081dabc5 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -159,7 +159,6 @@ MainWindow::MainWindow(FindObject * findObject, Camera * camera, QWidget * paren ui_->menuView->addAction(ui_->dockWidget_plot->toggleViewAction()); connect(ui_->toolBox, SIGNAL(parametersChanged(const QStringList &)), this, SLOT(notifyParametersChanged(const QStringList &))); - ui_->imageView_source->setGraphicsViewMode(true); ui_->imageView_source->setTextLabel(tr("Press \"space\" to start the camera or drop an image here...")); ui_->imageView_source->setMirrorView(Settings::getGeneral_mirrorView()); connect((QCheckBox*)ui_->toolBox->getParameterWidget(Settings::kGeneral_mirrorView()), @@ -1139,6 +1138,7 @@ void MainWindow::update(const cv::Mat & image) ui_->label_timeDetection->setNum(info.timeStamps_.value(DetectionInfo::kTimeKeypointDetection, 0)); ui_->label_timeSkewAffine->setNum(info.timeStamps_.value(DetectionInfo::kTimeSkewAffine, 0)); ui_->label_timeExtraction->setNum(info.timeStamps_.value(DetectionInfo::kTimeDescriptorExtraction, 0)); + ui_->label_timeSubPix->setNum(info.timeStamps_.value(DetectionInfo::kTimeSubPixelRefining, 0)); ui_->imageView_source->setData(info.sceneKeypoints_, cvtCvMat2QImage(sceneImage_)); if(!findObject_->vocabulary()->size()) { @@ -1339,6 +1339,7 @@ void MainWindow::update(const cv::Mat & image) ui_->label_timeDetection->setNum(info.timeStamps_.value(DetectionInfo::kTimeKeypointDetection, 0)); ui_->label_timeSkewAffine->setNum(info.timeStamps_.value(DetectionInfo::kTimeSkewAffine, 0)); ui_->label_timeExtraction->setNum(info.timeStamps_.value(DetectionInfo::kTimeDescriptorExtraction, 0)); + ui_->label_timeSubPix->setNum(info.timeStamps_.value(DetectionInfo::kTimeSubPixelRefining, 0)); ui_->imageView_source->setData(info.sceneKeypoints_, cvtCvMat2QImage(sceneImage_)); } diff --git a/src/ui/mainWindow.ui b/src/ui/mainWindow.ui index 7b6f6298..eee48ad0 100644 --- a/src/ui/mainWindow.ui +++ b/src/ui/mainWindow.ui @@ -7,7 +7,7 @@ 0 0 881 - 536 + 552 @@ -368,7 +368,7 @@ 0 0 222 - 409 + 425 @@ -475,84 +475,175 @@ 0 - + 000 - + 000 - + ms - + ms - + + + + ms + + + + + + + Vocabulary size + + + + + + + 000 + + + + + + + IP address + + + + + + + Output detection port + + + + + + + 0.0.0.0 + + + + + + + 0 + + + + + + + Objects detected + + + + + + + 000 + + + + + + + 000 + + + + + + + Input image port + + + + + + + - + + + + + + + Affine transforms + + + + Descriptors indexing - + ms - + 000 - + Min matched distance - + Max matched distance - + 000 - + Homograhies - + Descriptors extraction @@ -566,14 +657,14 @@ - + 000 - + ms @@ -594,7 +685,7 @@ - + Descriptors matching @@ -615,132 +706,62 @@ - - - - ms - - - - - - - Vocabulary size - - - - - - - 000 - - - - - - - IP address - - - - - - - Output detection port - - - - - - - 0.0.0.0 - - - - - - - 0 - - - - - - - Objects detected - - - - - - - 000 - - - - - - - 000 - - - - - - - Input image port - - - - - - - - - - - - - - - Affine transforms - - - - + 000 - + ms - + Refresh GUI - + 000 - + ms + + + + Features sub pixel refining + + + + + + + 000 + + + + + + + ms + + +