updated incremental vocabulary for binary descriptors (using BruteForce instead of LSH on descriptors not yet indexed)

added parameter General/vocabularyUpdateMinWords
renamed parameter General/incrementalVocabulary to General/vocabularyIncremental

git-svn-id: http://find-object.googlecode.com/svn/trunk/find_object@269 620bd6b2-0a58-f614-fd9a-1bd335dccda9

src/MainWindow.cpp

@@ -689,6 +689,9 @@ void MainWindow::updateObjects()
 {
 	if(objects_.size())
 	{
+		this->statusBar()->showMessage(tr("Updating %1 objects...").arg(objects_.size()));
+		QApplication::processEvents();
+
 		int threadCounts = Settings::getGeneral_threads();
 		if(threadCounts == 0)
 		{
@@ -788,6 +791,8 @@ void MainWindow::updateData()
 	// Copy data
 	if(count)
 	{
+		this->statusBar()->showMessage(tr("Updating objects data (%1 descriptors)...").arg(count));
+		QApplication::processEvents();
 		printf("Updating global descriptors matrix: Objects=%d, total descriptors=%d, dim=%d, type=%d\n", (int)objects_.size(), count, dim, type);
 		if(Settings::getGeneral_invertedSearch() || Settings::getGeneral_threads() == 1)
 		{
@@ -812,7 +817,7 @@ void MainWindow::updateData()
 			{
 				QTime time;
 				time.start();
-				bool incremental = Settings::getGeneral_incrementalVocabulary();
+				bool incremental = Settings::getGeneral_vocabularyIncremental();
 				if(incremental)
 				{
 					printf("Creating incremental vocabulary...\n");
@@ -823,18 +828,29 @@ void MainWindow::updateData()
 				}
 				QTime localTime;
 				localTime.start();
+				int updateVocabularyMinWords = Settings::getGeneral_vocabularyUpdateMinWords();
+				int addedWords = 0;
 				for(int i=0; i<objects_.size(); ++i)
 				{
 					QMultiMap<int, int> words = vocabulary_.addWords(objects_[i]->descriptors(), i, incremental);
 					objects_[i]->setWords(words);
-					printf("Object %d, %d words from %d descriptors (%d words, %d ms)\n",
+					addedWords += words.uniqueKeys().size();
+					bool updated = false;
+					if(incremental && addedWords > updateVocabularyMinWords)
+					{
+						vocabulary_.update();
+						addedWords = 0;
+						updated = true;
+					}
+					printf("Object %d, %d words from %d descriptors (%d words, %d ms) %s\n",
 							objects_[i]->id(),
 							words.uniqueKeys().size(),
 							objects_[i]->descriptors().rows,
 							vocabulary_.size(),
-							localTime.restart());
+							localTime.restart(),
+							updated?"updated":"");
 				}
-				if(!incremental)
+				if(addedWords)
 				{
 					vocabulary_.update();
 				}
@@ -857,6 +873,7 @@ void MainWindow::updateData()
 				objectsDescriptors_.push_back(objects_.at(i)->descriptors());
 			}
 		}
+		this->statusBar()->clearMessage();
 	}
 }
 
@@ -1230,8 +1247,8 @@ void MainWindow::update(const cv::Mat & image)
 				// CREATE INDEX for the scene
 				//printf("Creating FLANN index (%s)\n", Settings::currentNearestNeighborType().toStdString().c_str());
 				vocabulary_.clear();
-				QMultiMap<int, int> words = vocabulary_.addWords(descriptors, -1, Settings::getGeneral_incrementalVocabulary());
+				QMultiMap<int, int> words = vocabulary_.addWords(descriptors, -1, Settings::getGeneral_vocabularyIncremental());
-				if(!Settings::getGeneral_incrementalVocabulary())
+				if(!Settings::getGeneral_vocabularyIncremental())
 				{
 					vocabulary_.update();
 				}
@@ -1653,7 +1670,7 @@ void MainWindow::notifyParametersChanged(const QStringList & paramChanged)
 		else if(!nearestNeighborParamsChanged &&
 			    ( (iter->contains("NearestNeighbor") && Settings::getGeneral_invertedSearch()) ||
 			      iter->compare(Settings::kGeneral_invertedSearch()) == 0 ||
-			      (iter->compare(Settings::kGeneral_incrementalVocabulary()) == 0 && Settings::getGeneral_invertedSearch()) ||
+			      (iter->compare(Settings::kGeneral_vocabularyIncremental()) == 0 && Settings::getGeneral_invertedSearch()) ||
 			      (iter->compare(Settings::kGeneral_threads()) == 0 && !Settings::getGeneral_invertedSearch()) ))
 		{
 			nearestNeighborParamsChanged = true;

src/ParametersToolBox.cpp

@@ -471,14 +471,18 @@ void ParametersToolBox::changeParameter(const int & value)
 					QMessageBox::warning(this,
 							tr("Warning"),
 							tr("Current selected descriptor type (\"%1\") is binary while nearest neighbor strategy is not (\"%2\").\n"
-							   "Falling back to \"Lsh\" nearest neighbor strategy (by default).")
+							   "Falling back to \"Lsh\" nearest neighbor strategy with Hamming distance (by default).")
 							   .arg(descriptorBox->currentText())
 							   .arg(nnBox->currentText()));
 					QString tmp = Settings::getNearestNeighbor_1Strategy();
-					*tmp.begin() = '5'; // set index
+					*tmp.begin() = '5'; // set LSH
 					Settings::setNearestNeighbor_1Strategy(tmp);
+					tmp = Settings::getNearestNeighbor_2Distance_type();
+					*tmp.begin() = '8'; // set HAMMING
+					Settings::setNearestNeighbor_2Distance_type(tmp);
 					nnBox->blockSignals(true);
 					this->updateParameter(Settings::kNearestNeighbor_1Strategy());
+					this->updateParameter(Settings::kNearestNeighbor_2Distance_type());
 					nnBox->blockSignals(false);
 					if(sender() == nnBox)
 					{
@@ -493,14 +497,18 @@ void ParametersToolBox::changeParameter(const int & value)
 					QMessageBox::warning(this,
 							tr("Warning"),
 							tr("Current selected descriptor type (\"%1\") is not binary while nearest neighbor strategy is (\"%2\").\n"
-							   "Falling back to \"KDTree\" nearest neighbor strategy (by default).")
+							   "Falling back to \"KDTree\" nearest neighbor strategy with Euclidean_L2 distance (by default).")
 							   .arg(descriptorBox->currentText())
 							   .arg(nnBox->currentText()));
 					QString tmp = Settings::getNearestNeighbor_1Strategy();
-					*tmp.begin() = '1'; // set index
+					*tmp.begin() = '1'; // set KDTree
 					Settings::setNearestNeighbor_1Strategy(tmp);
+					tmp = Settings::getNearestNeighbor_2Distance_type();
+					*tmp.begin() = '0'; // set EUCLIDEAN_L2
+					Settings::setNearestNeighbor_2Distance_type(tmp);
 					nnBox->blockSignals(true);
 					this->updateParameter(Settings::kNearestNeighbor_1Strategy());
+					this->updateParameter(Settings::kNearestNeighbor_2Distance_type());
 					nnBox->blockSignals(false);
 					if(sender() == nnBox)
 					{

src/Settings.h

@@ -176,8 +176,9 @@ class Settings
 	PARAMETER(General, multiDetection, bool, false, "Multiple detection of the same object.");
 	PARAMETER(General, multiDetectionRadius, int, 30, "Ignore detection of the same object in X pixels radius of the previous detections.");
 	PARAMETER(General, port, int, 0, "Port on objects detected are published. If port=0, a port is chosen automatically.")
-	PARAMETER(General, incrementalVocabulary, bool, false, "The vocabulary is created incrementally. When new objects are added, their descriptors are compared to those already in vocabulary to find if the visual word already exist or not. \"NearestNeighbor/nndrRatio\" is used to compare descriptors.");
 	PARAMETER(General, autoScroll, bool, true, "Auto scroll to detected object in Objects panel.");
+	PARAMETER(General, vocabularyIncremental, bool, false, "The vocabulary is created incrementally. When new objects are added, their descriptors are compared to those already in vocabulary to find if the visual word already exist or not. \"NearestNeighbor/nndrRatio\" is used to compare descriptors.");
+	PARAMETER(General, vocabularyUpdateMinWords, int, 2000, "When the vocabulary is incremental (see \"General/vocabularyIncremental\"), after X words added to vocabulary, the internal index is updated with new words. This parameter lets avoiding to reconstruct the whole nearest neighbor index after each time descriptors of an object are added to vocabulary.");
 
 	PARAMETER(Homography, homographyComputed, bool, true, "Compute homography? On ROS, this is required to publish objects detected.");
 	PARAMETER(Homography, method, QString, "1:LMEDS;RANSAC", "Type of the robust estimation algorithm: least-median algorithm or RANSAC algorithm.");

src/Vocabulary.cpp

@@ -20,8 +20,10 @@ Vocabulary::~Vocabulary()
 
 void Vocabulary::clear()
 {
-	descriptors_ = cv::Mat();
+	indexedDescriptors_ = cv::Mat();
+	notIndexedDescriptors_ = cv::Mat();
 	wordToObjects_.clear();
+	notIndexedWordIds_.clear();
 }
 
 QMultiMap<int, int> Vocabulary::addWords(const cv::Mat & descriptors, int objectIndex, bool incremental)
@@ -35,43 +37,84 @@ QMultiMap<int, int> Vocabulary::addWords(const cv::Mat & descriptors, int object
 	if(incremental)
 	{
 		int k = 2;
-		cv::Mat results(descriptors.rows, k, CV_32SC1); // results index
+		cv::Mat results;
-		cv::Mat	dists(descriptors.rows, k, CV_32FC1); // Distance results are CV_32FC1
+		cv::Mat	dists;
 
 		bool globalSearch = false;
-		if(!descriptors_.empty() && descriptors_.rows >= (int)k)
+		if(!indexedDescriptors_.empty() && indexedDescriptors_.rows >= (int)k)
 		{
+			Q_ASSERT(indexedDescriptors_.type() == descriptors.type() && indexedDescriptors_.cols == descriptors.cols);
 			flannIndex_.knnSearch(descriptors, results, dists, k, Settings::getFlannSearchParams() );
+
+			if( dists.type() == CV_32S )
+			{
+				cv::Mat temp;
+				dists.convertTo(temp, CV_32F);
+				dists = temp;
+			}
+
 			globalSearch = true;
 		}
 
-		QVector<int> newWordsId; // index global
+		QVector<int> newWordIds = notIndexedWordIds_; // index global
-		cv::Mat newWords;
+		cv::Mat newWords = notIndexedDescriptors_;
 		int matches = 0;
 		for(int i = 0; i < descriptors.rows; ++i)
 		{
 			QMap<float, int> fullResults; // nearest descriptors sorted by distance
 			if(newWords.rows)
 			{
+				Q_ASSERT(newWords.type() == descriptors.type() && newWords.cols == descriptors.cols);
+
 				// Check if this descriptor matches with a word not already added to the vocabulary
-				cv::flann::Index tmpIndex;
+				// Do linear search only
-				cv::flann::IndexParams * params;
+				cv::Mat tmpResults;
+				cv::Mat	tmpDists;
 				if(descriptors.type()==CV_8U)
 				{
-					params = Settings::createFlannIndexParams(); // should be LSH
+					//normType – One of NORM_L1, NORM_L2, NORM_HAMMING, NORM_HAMMING2. L1 and L2 norms are
+					//			 preferable choices for SIFT and SURF descriptors, NORM_HAMMING should be
+					// 			 used with ORB, BRISK and BRIEF, NORM_HAMMING2 should be used with ORB
+					// 			 when WTA_K==3 or 4 (see ORB::ORB constructor description).
+					int normType = cv::NORM_HAMMING;
+					if(Settings::currentDescriptorType().compare("ORB") &&
+						(Settings::getFeature2D_ORB_WTA_K()==3 || Settings::getFeature2D_ORB_WTA_K()==4))
+					{
+						normType = cv::NORM_HAMMING2;
+					}
+
+					cv::batchDistance( descriptors.row(i),
+									newWords,
+									tmpDists,
+									CV_32S,
+									tmpResults,
+									normType,
+									newWords.rows>=k?k:1,
+									cv::Mat(),
+									0,
+									false);
 				}
 				else
 				{
-					params = new cv::flann::LinearIndexParams(); // faster
+					cv::flann::Index tmpIndex;
+					tmpIndex.build(newWords, cv::flann::LinearIndexParams(), Settings::getFlannDistanceType());
+					tmpIndex.knnSearch(descriptors.row(i), tmpResults, tmpDists, newWords.rows>1?k:1, Settings::getFlannSearchParams());
 				}
-				tmpIndex.build(newWords, *params, Settings::getFlannDistanceType());
+
-				delete params;
+				if( tmpDists.type() == CV_32S )
-				cv::Mat tmpResults(1, newWords.rows>1?k:1, CV_32SC1); // results index
-				cv::Mat	tmpDists(1, newWords.rows>1?k:1, CV_32FC1); // Distance results are CV_32FC1
-				tmpIndex.knnSearch(descriptors.row(i), tmpResults, tmpDists, newWords.rows>1?k:1, Settings::getFlannSearchParams());
-				for(int j = 0; j < (newWords.rows>1?k:1); ++j)
 				{
-					fullResults.insert(tmpDists.at<float>(0,j), newWordsId.at(tmpResults.at<int>(0,j)));
+					cv::Mat temp;
+					tmpDists.convertTo(temp, CV_32F);
+					tmpDists = temp;
+				}
+
+				for(int j = 0; j < (newWords.rows>=k?k:1); ++j)
+				{
+					if(tmpResults.at<int>(0,j) >= 0)
+					{
+						//printf("local i=%d, j=%d, tmpDist=%f tmpResult=%d\n", i ,j, tmpDists.at<float>(0,j), tmpResults.at<int>(0,j));
+						fullResults.insert(tmpDists.at<float>(0,j), newWordIds.at(tmpResults.at<int>(0,j)));
+					}
 				}
 			}
 
@@ -79,7 +122,11 @@ QMultiMap<int, int> Vocabulary::addWords(const cv::Mat & descriptors, int object
 			{
 				for(int j=0; j<k; ++j)
 				{
-					fullResults.insert(dists.at<float>(i,j), results.at<int>(i,j));
+					if(results.at<int>(i,j) >= 0)
+					{
+						//printf("global i=%d, j=%d, dist=%f\n", i ,j, dists.at<float>(i,j));
+						fullResults.insert(dists.at<float>(i,j), results.at<int>(i,j));
+					}
 				}
 			}
 
@@ -107,49 +154,39 @@ QMultiMap<int, int> Vocabulary::addWords(const cv::Mat & descriptors, int object
 				}
 				cv::Mat dest(tmp, cv::Range(newWords.rows, newWords.rows+1));
 				descriptors.row(i).copyTo(dest);
-				newWordsId.push_back(descriptors_.rows + newWords.rows);
+				newWordIds.push_back(indexedDescriptors_.rows + newWords.rows);
 				newWords = tmp;
-				words.insert(newWordsId.back(), i);
+				words.insert(newWordIds.back(), i);
-				wordToObjects_.insert(newWordsId.back(), objectIndex);
+				wordToObjects_.insert(newWordIds.back(), objectIndex);
 			}
 		}
-		//printf("matches = %d\n", matches);
 
 		//concatenate new words
 		if(newWords.rows)
 		{
-			cv::Mat tmp(descriptors_.rows+newWords.rows, descriptors.cols, descriptors.type());
+			notIndexedWordIds_ = newWordIds;
-			if(descriptors_.rows)
+			notIndexedDescriptors_ = newWords;
-			{
-				cv::Mat dest(tmp, cv::Range(0, descriptors_.rows));
-				descriptors_.copyTo(dest);
-			}
-			cv::Mat dest(tmp, cv::Range(descriptors_.rows, descriptors_.rows+newWords.rows));
-			newWords.copyTo(dest);
-			descriptors_ = tmp;
 		}
-
-		//update
-		this->update();
 	}
 	else
 	{
 		for(int i = 0; i < descriptors.rows; ++i)
 		{
-			wordToObjects_.insert(descriptors_.rows+i, objectIndex);
+			wordToObjects_.insert(indexedDescriptors_.rows + notIndexedDescriptors_.rows+i, objectIndex);
-			words.insert(descriptors_.rows+i, i);
+			words.insert(indexedDescriptors_.rows + notIndexedDescriptors_.rows+i, i);
+			notIndexedWordIds_.push_back(indexedDescriptors_.rows + notIndexedDescriptors_.rows+i);
 		}
 
 		//just concatenate descriptors
-		cv::Mat tmp(descriptors_.rows+descriptors.rows, descriptors.cols, descriptors.type());
+		cv::Mat tmp(notIndexedDescriptors_.rows+descriptors.rows, descriptors.cols, descriptors.type());
-		if(descriptors_.rows)
+		if(notIndexedDescriptors_.rows)
 		{
-			cv::Mat dest(tmp, cv::Range(0, descriptors_.rows));
+			cv::Mat dest(tmp, cv::Range(0, notIndexedDescriptors_.rows));
-			descriptors_.copyTo(dest);
+			notIndexedDescriptors_.copyTo(dest);
 		}
-		cv::Mat dest(tmp, cv::Range(descriptors_.rows, descriptors_.rows+descriptors.rows));
+		cv::Mat dest(tmp, cv::Range(notIndexedDescriptors_.rows, notIndexedDescriptors_.rows+descriptors.rows));
 		descriptors.copyTo(dest);
-		descriptors_ = tmp;
+		notIndexedDescriptors_ = tmp;
 	}
 
 	return words;
@@ -157,18 +194,46 @@ QMultiMap<int, int> Vocabulary::addWords(const cv::Mat & descriptors, int object
 
 void Vocabulary::update()
 {
-	if(!descriptors_.empty())
+	if(!notIndexedDescriptors_.empty())
+	{
+		Q_ASSERT(indexedDescriptors_.cols == notIndexedDescriptors_.cols &&
+				 indexedDescriptors_.type() == notIndexedDescriptors_.type() );
+
+		//concatenate descriptors
+		cv::Mat tmp(indexedDescriptors_.rows+notIndexedDescriptors_.rows, notIndexedDescriptors_.cols, notIndexedDescriptors_.type());
+		cv::Mat dest(tmp, cv::Range(0, indexedDescriptors_.rows));
+		indexedDescriptors_.copyTo(dest);
+		dest = cv::Mat(tmp, cv::Range(indexedDescriptors_.rows, indexedDescriptors_.rows+notIndexedDescriptors_.rows));
+		notIndexedDescriptors_.copyTo(dest);
+		indexedDescriptors_ = tmp;
+
+		notIndexedDescriptors_ = cv::Mat();
+		notIndexedWordIds_.clear();
+	}
+
+	if(!indexedDescriptors_.empty())
 	{
 		cv::flann::IndexParams * params = Settings::createFlannIndexParams();
-		flannIndex_.build(descriptors_, *params, Settings::getFlannDistanceType());
+		flannIndex_.build(indexedDescriptors_, *params, Settings::getFlannDistanceType());
 		delete params;
 	}
 }
 
 void Vocabulary::search(const cv::Mat & descriptors, cv::Mat & results, cv::Mat & dists, int k)
 {
-	if(!descriptors_.empty())
+	Q_ASSERT(notIndexedDescriptors_.empty() && notIndexedWordIds_.size() == 0);
+
+	if(!indexedDescriptors_.empty())
 	{
+		Q_ASSERT(descriptors.type() == indexedDescriptors_.type() && descriptors.cols == indexedDescriptors_.cols);
+
 		flannIndex_.knnSearch(descriptors, results, dists, k, Settings::getFlannSearchParams());
+
+		if( dists.type() == CV_32S )
+		{
+			cv::Mat temp;
+			dists.convertTo(temp, CV_32F);
+			dists = temp;
+		}
 	}
 }

src/Vocabulary.h

@@ -9,6 +9,7 @@
 #define VOCABULARY_H_
 
 #include <QtCore/QMultiMap>
+#include <QtCore/QVector>
 #include <opencv2/opencv.hpp>
 
 class Vocabulary {
@@ -20,13 +21,15 @@ public:
 	QMultiMap<int, int> addWords(const cv::Mat & descriptors, int objectIndex, bool incremental);
 	void update();
 	void search(const cv::Mat & descriptors, cv::Mat & results, cv::Mat & dists, int k);
-	int size() const {return descriptors_.rows;}
+	int size() const {return indexedDescriptors_.rows + notIndexedDescriptors_.rows;}
 	const QMultiMap<int, int> & wordToObjects() const {return wordToObjects_;}
 
 private:
 	cv::flann::Index flannIndex_;
-	cv::Mat descriptors_;
+	cv::Mat indexedDescriptors_;
+	cv::Mat notIndexedDescriptors_;
 	QMultiMap<int, int> wordToObjects_;
+	QVector<int> notIndexedWordIds_;
 };
 
 #endif /* VOCABULARY_H_ */