From 20a32932bfe4167bc10d2f4eb9b0d6967a72bb78 Mon Sep 17 00:00:00 2001 From: matlabbe Date: Mon, 12 May 2014 22:58:57 +0000 Subject: [PATCH] updated incremental vocabulary for binary descriptors (using BruteForce instead of LSH on descriptors not yet indexed) added parameter General/vocabularyUpdateMinWords renamed parameter General/incrementalVocabulary to General/vocabularyIncremental git-svn-id: http://find-object.googlecode.com/svn/trunk/find_object@269 620bd6b2-0a58-f614-fd9a-1bd335dccda9 --- src/MainWindow.cpp | 31 ++++++-- src/ParametersToolBox.cpp | 16 +++- src/Settings.h | 3 +- src/Vocabulary.cpp | 155 +++++++++++++++++++++++++++----------- src/Vocabulary.h | 7 +- 5 files changed, 153 insertions(+), 59 deletions(-) diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index 7523f977..727b51c9 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -689,6 +689,9 @@ void MainWindow::updateObjects() { if(objects_.size()) { + this->statusBar()->showMessage(tr("Updating %1 objects...").arg(objects_.size())); + QApplication::processEvents(); + int threadCounts = Settings::getGeneral_threads(); if(threadCounts == 0) { @@ -788,6 +791,8 @@ void MainWindow::updateData() // Copy data if(count) { + this->statusBar()->showMessage(tr("Updating objects data (%1 descriptors)...").arg(count)); + QApplication::processEvents(); printf("Updating global descriptors matrix: Objects=%d, total descriptors=%d, dim=%d, type=%d\n", (int)objects_.size(), count, dim, type); if(Settings::getGeneral_invertedSearch() || Settings::getGeneral_threads() == 1) { @@ -812,7 +817,7 @@ void MainWindow::updateData() { QTime time; time.start(); - bool incremental = Settings::getGeneral_incrementalVocabulary(); + bool incremental = Settings::getGeneral_vocabularyIncremental(); if(incremental) { printf("Creating incremental vocabulary...\n"); @@ -823,18 +828,29 @@ void MainWindow::updateData() } QTime localTime; localTime.start(); + int updateVocabularyMinWords = Settings::getGeneral_vocabularyUpdateMinWords(); + int addedWords = 0; for(int i=0; i words = vocabulary_.addWords(objects_[i]->descriptors(), i, incremental); objects_[i]->setWords(words); - printf("Object %d, %d words from %d descriptors (%d words, %d ms)\n", + addedWords += words.uniqueKeys().size(); + bool updated = false; + if(incremental && addedWords > updateVocabularyMinWords) + { + vocabulary_.update(); + addedWords = 0; + updated = true; + } + printf("Object %d, %d words from %d descriptors (%d words, %d ms) %s\n", objects_[i]->id(), words.uniqueKeys().size(), objects_[i]->descriptors().rows, vocabulary_.size(), - localTime.restart()); + localTime.restart(), + updated?"updated":""); } - if(!incremental) + if(addedWords) { vocabulary_.update(); } @@ -857,6 +873,7 @@ void MainWindow::updateData() objectsDescriptors_.push_back(objects_.at(i)->descriptors()); } } + this->statusBar()->clearMessage(); } } @@ -1230,8 +1247,8 @@ void MainWindow::update(const cv::Mat & image) // CREATE INDEX for the scene //printf("Creating FLANN index (%s)\n", Settings::currentNearestNeighborType().toStdString().c_str()); vocabulary_.clear(); - QMultiMap words = vocabulary_.addWords(descriptors, -1, Settings::getGeneral_incrementalVocabulary()); - if(!Settings::getGeneral_incrementalVocabulary()) + QMultiMap words = vocabulary_.addWords(descriptors, -1, Settings::getGeneral_vocabularyIncremental()); + if(!Settings::getGeneral_vocabularyIncremental()) { vocabulary_.update(); } @@ -1653,7 +1670,7 @@ void MainWindow::notifyParametersChanged(const QStringList & paramChanged) else if(!nearestNeighborParamsChanged && ( (iter->contains("NearestNeighbor") && Settings::getGeneral_invertedSearch()) || iter->compare(Settings::kGeneral_invertedSearch()) == 0 || - (iter->compare(Settings::kGeneral_incrementalVocabulary()) == 0 && Settings::getGeneral_invertedSearch()) || + (iter->compare(Settings::kGeneral_vocabularyIncremental()) == 0 && Settings::getGeneral_invertedSearch()) || (iter->compare(Settings::kGeneral_threads()) == 0 && !Settings::getGeneral_invertedSearch()) )) { nearestNeighborParamsChanged = true; diff --git a/src/ParametersToolBox.cpp b/src/ParametersToolBox.cpp index 32770033..7518da5f 100644 --- a/src/ParametersToolBox.cpp +++ b/src/ParametersToolBox.cpp @@ -471,14 +471,18 @@ void ParametersToolBox::changeParameter(const int & value) QMessageBox::warning(this, tr("Warning"), tr("Current selected descriptor type (\"%1\") is binary while nearest neighbor strategy is not (\"%2\").\n" - "Falling back to \"Lsh\" nearest neighbor strategy (by default).") + "Falling back to \"Lsh\" nearest neighbor strategy with Hamming distance (by default).") .arg(descriptorBox->currentText()) .arg(nnBox->currentText())); QString tmp = Settings::getNearestNeighbor_1Strategy(); - *tmp.begin() = '5'; // set index + *tmp.begin() = '5'; // set LSH Settings::setNearestNeighbor_1Strategy(tmp); + tmp = Settings::getNearestNeighbor_2Distance_type(); + *tmp.begin() = '8'; // set HAMMING + Settings::setNearestNeighbor_2Distance_type(tmp); nnBox->blockSignals(true); this->updateParameter(Settings::kNearestNeighbor_1Strategy()); + this->updateParameter(Settings::kNearestNeighbor_2Distance_type()); nnBox->blockSignals(false); if(sender() == nnBox) { @@ -493,14 +497,18 @@ void ParametersToolBox::changeParameter(const int & value) QMessageBox::warning(this, tr("Warning"), tr("Current selected descriptor type (\"%1\") is not binary while nearest neighbor strategy is (\"%2\").\n" - "Falling back to \"KDTree\" nearest neighbor strategy (by default).") + "Falling back to \"KDTree\" nearest neighbor strategy with Euclidean_L2 distance (by default).") .arg(descriptorBox->currentText()) .arg(nnBox->currentText())); QString tmp = Settings::getNearestNeighbor_1Strategy(); - *tmp.begin() = '1'; // set index + *tmp.begin() = '1'; // set KDTree Settings::setNearestNeighbor_1Strategy(tmp); + tmp = Settings::getNearestNeighbor_2Distance_type(); + *tmp.begin() = '0'; // set EUCLIDEAN_L2 + Settings::setNearestNeighbor_2Distance_type(tmp); nnBox->blockSignals(true); this->updateParameter(Settings::kNearestNeighbor_1Strategy()); + this->updateParameter(Settings::kNearestNeighbor_2Distance_type()); nnBox->blockSignals(false); if(sender() == nnBox) { diff --git a/src/Settings.h b/src/Settings.h index 7e7c62f2..a4f7f746 100644 --- a/src/Settings.h +++ b/src/Settings.h @@ -176,8 +176,9 @@ class Settings PARAMETER(General, multiDetection, bool, false, "Multiple detection of the same object."); PARAMETER(General, multiDetectionRadius, int, 30, "Ignore detection of the same object in X pixels radius of the previous detections."); PARAMETER(General, port, int, 0, "Port on objects detected are published. If port=0, a port is chosen automatically.") - PARAMETER(General, incrementalVocabulary, bool, false, "The vocabulary is created incrementally. When new objects are added, their descriptors are compared to those already in vocabulary to find if the visual word already exist or not. \"NearestNeighbor/nndrRatio\" is used to compare descriptors."); PARAMETER(General, autoScroll, bool, true, "Auto scroll to detected object in Objects panel."); + PARAMETER(General, vocabularyIncremental, bool, false, "The vocabulary is created incrementally. When new objects are added, their descriptors are compared to those already in vocabulary to find if the visual word already exist or not. \"NearestNeighbor/nndrRatio\" is used to compare descriptors."); + PARAMETER(General, vocabularyUpdateMinWords, int, 2000, "When the vocabulary is incremental (see \"General/vocabularyIncremental\"), after X words added to vocabulary, the internal index is updated with new words. This parameter lets avoiding to reconstruct the whole nearest neighbor index after each time descriptors of an object are added to vocabulary."); PARAMETER(Homography, homographyComputed, bool, true, "Compute homography? On ROS, this is required to publish objects detected."); PARAMETER(Homography, method, QString, "1:LMEDS;RANSAC", "Type of the robust estimation algorithm: least-median algorithm or RANSAC algorithm."); diff --git a/src/Vocabulary.cpp b/src/Vocabulary.cpp index b97e0375..82ce4838 100644 --- a/src/Vocabulary.cpp +++ b/src/Vocabulary.cpp @@ -20,8 +20,10 @@ Vocabulary::~Vocabulary() void Vocabulary::clear() { - descriptors_ = cv::Mat(); + indexedDescriptors_ = cv::Mat(); + notIndexedDescriptors_ = cv::Mat(); wordToObjects_.clear(); + notIndexedWordIds_.clear(); } QMultiMap Vocabulary::addWords(const cv::Mat & descriptors, int objectIndex, bool incremental) @@ -35,43 +37,84 @@ QMultiMap Vocabulary::addWords(const cv::Mat & descriptors, int object if(incremental) { int k = 2; - cv::Mat results(descriptors.rows, k, CV_32SC1); // results index - cv::Mat dists(descriptors.rows, k, CV_32FC1); // Distance results are CV_32FC1 + cv::Mat results; + cv::Mat dists; bool globalSearch = false; - if(!descriptors_.empty() && descriptors_.rows >= (int)k) + if(!indexedDescriptors_.empty() && indexedDescriptors_.rows >= (int)k) { + Q_ASSERT(indexedDescriptors_.type() == descriptors.type() && indexedDescriptors_.cols == descriptors.cols); flannIndex_.knnSearch(descriptors, results, dists, k, Settings::getFlannSearchParams() ); + + if( dists.type() == CV_32S ) + { + cv::Mat temp; + dists.convertTo(temp, CV_32F); + dists = temp; + } + globalSearch = true; } - QVector newWordsId; // index global - cv::Mat newWords; + QVector newWordIds = notIndexedWordIds_; // index global + cv::Mat newWords = notIndexedDescriptors_; int matches = 0; for(int i = 0; i < descriptors.rows; ++i) { QMap fullResults; // nearest descriptors sorted by distance if(newWords.rows) { + Q_ASSERT(newWords.type() == descriptors.type() && newWords.cols == descriptors.cols); + // Check if this descriptor matches with a word not already added to the vocabulary - cv::flann::Index tmpIndex; - cv::flann::IndexParams * params; + // Do linear search only + cv::Mat tmpResults; + cv::Mat tmpDists; if(descriptors.type()==CV_8U) { - params = Settings::createFlannIndexParams(); // should be LSH + //normType – One of NORM_L1, NORM_L2, NORM_HAMMING, NORM_HAMMING2. L1 and L2 norms are + // preferable choices for SIFT and SURF descriptors, NORM_HAMMING should be + // used with ORB, BRISK and BRIEF, NORM_HAMMING2 should be used with ORB + // when WTA_K==3 or 4 (see ORB::ORB constructor description). + int normType = cv::NORM_HAMMING; + if(Settings::currentDescriptorType().compare("ORB") && + (Settings::getFeature2D_ORB_WTA_K()==3 || Settings::getFeature2D_ORB_WTA_K()==4)) + { + normType = cv::NORM_HAMMING2; + } + + cv::batchDistance( descriptors.row(i), + newWords, + tmpDists, + CV_32S, + tmpResults, + normType, + newWords.rows>=k?k:1, + cv::Mat(), + 0, + false); } else { - params = new cv::flann::LinearIndexParams(); // faster + cv::flann::Index tmpIndex; + tmpIndex.build(newWords, cv::flann::LinearIndexParams(), Settings::getFlannDistanceType()); + tmpIndex.knnSearch(descriptors.row(i), tmpResults, tmpDists, newWords.rows>1?k:1, Settings::getFlannSearchParams()); } - tmpIndex.build(newWords, *params, Settings::getFlannDistanceType()); - delete params; - cv::Mat tmpResults(1, newWords.rows>1?k:1, CV_32SC1); // results index - cv::Mat tmpDists(1, newWords.rows>1?k:1, CV_32FC1); // Distance results are CV_32FC1 - tmpIndex.knnSearch(descriptors.row(i), tmpResults, tmpDists, newWords.rows>1?k:1, Settings::getFlannSearchParams()); - for(int j = 0; j < (newWords.rows>1?k:1); ++j) + + if( tmpDists.type() == CV_32S ) { - fullResults.insert(tmpDists.at(0,j), newWordsId.at(tmpResults.at(0,j))); + cv::Mat temp; + tmpDists.convertTo(temp, CV_32F); + tmpDists = temp; + } + + for(int j = 0; j < (newWords.rows>=k?k:1); ++j) + { + if(tmpResults.at(0,j) >= 0) + { + //printf("local i=%d, j=%d, tmpDist=%f tmpResult=%d\n", i ,j, tmpDists.at(0,j), tmpResults.at(0,j)); + fullResults.insert(tmpDists.at(0,j), newWordIds.at(tmpResults.at(0,j))); + } } } @@ -79,7 +122,11 @@ QMultiMap Vocabulary::addWords(const cv::Mat & descriptors, int object { for(int j=0; j(i,j), results.at(i,j)); + if(results.at(i,j) >= 0) + { + //printf("global i=%d, j=%d, dist=%f\n", i ,j, dists.at(i,j)); + fullResults.insert(dists.at(i,j), results.at(i,j)); + } } } @@ -107,49 +154,39 @@ QMultiMap Vocabulary::addWords(const cv::Mat & descriptors, int object } cv::Mat dest(tmp, cv::Range(newWords.rows, newWords.rows+1)); descriptors.row(i).copyTo(dest); - newWordsId.push_back(descriptors_.rows + newWords.rows); + newWordIds.push_back(indexedDescriptors_.rows + newWords.rows); newWords = tmp; - words.insert(newWordsId.back(), i); - wordToObjects_.insert(newWordsId.back(), objectIndex); + words.insert(newWordIds.back(), i); + wordToObjects_.insert(newWordIds.back(), objectIndex); } } - //printf("matches = %d\n", matches); //concatenate new words if(newWords.rows) { - cv::Mat tmp(descriptors_.rows+newWords.rows, descriptors.cols, descriptors.type()); - if(descriptors_.rows) - { - cv::Mat dest(tmp, cv::Range(0, descriptors_.rows)); - descriptors_.copyTo(dest); - } - cv::Mat dest(tmp, cv::Range(descriptors_.rows, descriptors_.rows+newWords.rows)); - newWords.copyTo(dest); - descriptors_ = tmp; + notIndexedWordIds_ = newWordIds; + notIndexedDescriptors_ = newWords; } - - //update - this->update(); } else { for(int i = 0; i < descriptors.rows; ++i) { - wordToObjects_.insert(descriptors_.rows+i, objectIndex); - words.insert(descriptors_.rows+i, i); + wordToObjects_.insert(indexedDescriptors_.rows + notIndexedDescriptors_.rows+i, objectIndex); + words.insert(indexedDescriptors_.rows + notIndexedDescriptors_.rows+i, i); + notIndexedWordIds_.push_back(indexedDescriptors_.rows + notIndexedDescriptors_.rows+i); } //just concatenate descriptors - cv::Mat tmp(descriptors_.rows+descriptors.rows, descriptors.cols, descriptors.type()); - if(descriptors_.rows) + cv::Mat tmp(notIndexedDescriptors_.rows+descriptors.rows, descriptors.cols, descriptors.type()); + if(notIndexedDescriptors_.rows) { - cv::Mat dest(tmp, cv::Range(0, descriptors_.rows)); - descriptors_.copyTo(dest); + cv::Mat dest(tmp, cv::Range(0, notIndexedDescriptors_.rows)); + notIndexedDescriptors_.copyTo(dest); } - cv::Mat dest(tmp, cv::Range(descriptors_.rows, descriptors_.rows+descriptors.rows)); + cv::Mat dest(tmp, cv::Range(notIndexedDescriptors_.rows, notIndexedDescriptors_.rows+descriptors.rows)); descriptors.copyTo(dest); - descriptors_ = tmp; + notIndexedDescriptors_ = tmp; } return words; @@ -157,18 +194,46 @@ QMultiMap Vocabulary::addWords(const cv::Mat & descriptors, int object void Vocabulary::update() { - if(!descriptors_.empty()) + if(!notIndexedDescriptors_.empty()) + { + Q_ASSERT(indexedDescriptors_.cols == notIndexedDescriptors_.cols && + indexedDescriptors_.type() == notIndexedDescriptors_.type() ); + + //concatenate descriptors + cv::Mat tmp(indexedDescriptors_.rows+notIndexedDescriptors_.rows, notIndexedDescriptors_.cols, notIndexedDescriptors_.type()); + cv::Mat dest(tmp, cv::Range(0, indexedDescriptors_.rows)); + indexedDescriptors_.copyTo(dest); + dest = cv::Mat(tmp, cv::Range(indexedDescriptors_.rows, indexedDescriptors_.rows+notIndexedDescriptors_.rows)); + notIndexedDescriptors_.copyTo(dest); + indexedDescriptors_ = tmp; + + notIndexedDescriptors_ = cv::Mat(); + notIndexedWordIds_.clear(); + } + + if(!indexedDescriptors_.empty()) { cv::flann::IndexParams * params = Settings::createFlannIndexParams(); - flannIndex_.build(descriptors_, *params, Settings::getFlannDistanceType()); + flannIndex_.build(indexedDescriptors_, *params, Settings::getFlannDistanceType()); delete params; } } void Vocabulary::search(const cv::Mat & descriptors, cv::Mat & results, cv::Mat & dists, int k) { - if(!descriptors_.empty()) + Q_ASSERT(notIndexedDescriptors_.empty() && notIndexedWordIds_.size() == 0); + + if(!indexedDescriptors_.empty()) { + Q_ASSERT(descriptors.type() == indexedDescriptors_.type() && descriptors.cols == indexedDescriptors_.cols); + flannIndex_.knnSearch(descriptors, results, dists, k, Settings::getFlannSearchParams()); + + if( dists.type() == CV_32S ) + { + cv::Mat temp; + dists.convertTo(temp, CV_32F); + dists = temp; + } } } diff --git a/src/Vocabulary.h b/src/Vocabulary.h index f586468f..a236a850 100644 --- a/src/Vocabulary.h +++ b/src/Vocabulary.h @@ -9,6 +9,7 @@ #define VOCABULARY_H_ #include +#include #include class Vocabulary { @@ -20,13 +21,15 @@ public: QMultiMap addWords(const cv::Mat & descriptors, int objectIndex, bool incremental); void update(); void search(const cv::Mat & descriptors, cv::Mat & results, cv::Mat & dists, int k); - int size() const {return descriptors_.rows;} + int size() const {return indexedDescriptors_.rows + notIndexedDescriptors_.rows;} const QMultiMap & wordToObjects() const {return wordToObjects_;} private: cv::flann::Index flannIndex_; - cv::Mat descriptors_; + cv::Mat indexedDescriptors_; + cv::Mat notIndexedDescriptors_; QMultiMap wordToObjects_; + QVector notIndexedWordIds_; }; #endif /* VOCABULARY_H_ */