updated incremental vocabulary for binary descriptors (using BruteForce instead of LSH on descriptors not yet indexed)
added parameter General/vocabularyUpdateMinWords renamed parameter General/incrementalVocabulary to General/vocabularyIncremental git-svn-id: http://find-object.googlecode.com/svn/trunk/find_object@269 620bd6b2-0a58-f614-fd9a-1bd335dccda9
This commit is contained in:
matlabbe
parent
228cce72bf
commit
20a32932bf
@ -689,6 +689,9 @@ void MainWindow::updateObjects()
|
||||
{
|
||||
if(objects_.size())
|
||||
{
|
||||
this->statusBar()->showMessage(tr("Updating %1 objects...").arg(objects_.size()));
|
||||
QApplication::processEvents();
|
||||
|
||||
int threadCounts = Settings::getGeneral_threads();
|
||||
if(threadCounts == 0)
|
||||
{
|
||||
@ -788,6 +791,8 @@ void MainWindow::updateData()
|
||||
// Copy data
|
||||
if(count)
|
||||
{
|
||||
this->statusBar()->showMessage(tr("Updating objects data (%1 descriptors)...").arg(count));
|
||||
QApplication::processEvents();
|
||||
printf("Updating global descriptors matrix: Objects=%d, total descriptors=%d, dim=%d, type=%d\n", (int)objects_.size(), count, dim, type);
|
||||
if(Settings::getGeneral_invertedSearch() || Settings::getGeneral_threads() == 1)
|
||||
{
|
||||
@ -812,7 +817,7 @@ void MainWindow::updateData()
|
||||
{
|
||||
QTime time;
|
||||
time.start();
|
||||
bool incremental = Settings::getGeneral_incrementalVocabulary();
|
||||
bool incremental = Settings::getGeneral_vocabularyIncremental();
|
||||
if(incremental)
|
||||
{
|
||||
printf("Creating incremental vocabulary...\n");
|
||||
@ -823,18 +828,29 @@ void MainWindow::updateData()
|
||||
}
|
||||
QTime localTime;
|
||||
localTime.start();
|
||||
int updateVocabularyMinWords = Settings::getGeneral_vocabularyUpdateMinWords();
|
||||
int addedWords = 0;
|
||||
for(int i=0; i<objects_.size(); ++i)
|
||||
{
|
||||
QMultiMap<int, int> words = vocabulary_.addWords(objects_[i]->descriptors(), i, incremental);
|
||||
objects_[i]->setWords(words);
|
||||
printf("Object %d, %d words from %d descriptors (%d words, %d ms)\n",
|
||||
addedWords += words.uniqueKeys().size();
|
||||
bool updated = false;
|
||||
if(incremental && addedWords > updateVocabularyMinWords)
|
||||
{
|
||||
vocabulary_.update();
|
||||
addedWords = 0;
|
||||
updated = true;
|
||||
}
|
||||
printf("Object %d, %d words from %d descriptors (%d words, %d ms) %s\n",
|
||||
objects_[i]->id(),
|
||||
words.uniqueKeys().size(),
|
||||
objects_[i]->descriptors().rows,
|
||||
vocabulary_.size(),
|
||||
localTime.restart());
|
||||
localTime.restart(),
|
||||
updated?"updated":"");
|
||||
}
|
||||
if(!incremental)
|
||||
if(addedWords)
|
||||
{
|
||||
vocabulary_.update();
|
||||
}
|
||||
@ -857,6 +873,7 @@ void MainWindow::updateData()
|
||||
objectsDescriptors_.push_back(objects_.at(i)->descriptors());
|
||||
}
|
||||
}
|
||||
this->statusBar()->clearMessage();
|
||||
}
|
||||
}
|
||||
|
||||
@ -1230,8 +1247,8 @@ void MainWindow::update(const cv::Mat & image)
|
||||
// CREATE INDEX for the scene
|
||||
//printf("Creating FLANN index (%s)\n", Settings::currentNearestNeighborType().toStdString().c_str());
|
||||
vocabulary_.clear();
|
||||
QMultiMap<int, int> words = vocabulary_.addWords(descriptors, -1, Settings::getGeneral_incrementalVocabulary());
|
||||
if(!Settings::getGeneral_incrementalVocabulary())
|
||||
QMultiMap<int, int> words = vocabulary_.addWords(descriptors, -1, Settings::getGeneral_vocabularyIncremental());
|
||||
if(!Settings::getGeneral_vocabularyIncremental())
|
||||
{
|
||||
vocabulary_.update();
|
||||
}
|
||||
@ -1653,7 +1670,7 @@ void MainWindow::notifyParametersChanged(const QStringList & paramChanged)
|
||||
else if(!nearestNeighborParamsChanged &&
|
||||
( (iter->contains("NearestNeighbor") && Settings::getGeneral_invertedSearch()) ||
|
||||
iter->compare(Settings::kGeneral_invertedSearch()) == 0 ||
|
||||
(iter->compare(Settings::kGeneral_incrementalVocabulary()) == 0 && Settings::getGeneral_invertedSearch()) ||
|
||||
(iter->compare(Settings::kGeneral_vocabularyIncremental()) == 0 && Settings::getGeneral_invertedSearch()) ||
|
||||
(iter->compare(Settings::kGeneral_threads()) == 0 && !Settings::getGeneral_invertedSearch()) ))
|
||||
{
|
||||
nearestNeighborParamsChanged = true;
|
||||
|
||||
@ -471,14 +471,18 @@ void ParametersToolBox::changeParameter(const int & value)
|
||||
QMessageBox::warning(this,
|
||||
tr("Warning"),
|
||||
tr("Current selected descriptor type (\"%1\") is binary while nearest neighbor strategy is not (\"%2\").\n"
|
||||
"Falling back to \"Lsh\" nearest neighbor strategy (by default).")
|
||||
"Falling back to \"Lsh\" nearest neighbor strategy with Hamming distance (by default).")
|
||||
.arg(descriptorBox->currentText())
|
||||
.arg(nnBox->currentText()));
|
||||
QString tmp = Settings::getNearestNeighbor_1Strategy();
|
||||
*tmp.begin() = '5'; // set index
|
||||
*tmp.begin() = '5'; // set LSH
|
||||
Settings::setNearestNeighbor_1Strategy(tmp);
|
||||
tmp = Settings::getNearestNeighbor_2Distance_type();
|
||||
*tmp.begin() = '8'; // set HAMMING
|
||||
Settings::setNearestNeighbor_2Distance_type(tmp);
|
||||
nnBox->blockSignals(true);
|
||||
this->updateParameter(Settings::kNearestNeighbor_1Strategy());
|
||||
this->updateParameter(Settings::kNearestNeighbor_2Distance_type());
|
||||
nnBox->blockSignals(false);
|
||||
if(sender() == nnBox)
|
||||
{
|
||||
@ -493,14 +497,18 @@ void ParametersToolBox::changeParameter(const int & value)
|
||||
QMessageBox::warning(this,
|
||||
tr("Warning"),
|
||||
tr("Current selected descriptor type (\"%1\") is not binary while nearest neighbor strategy is (\"%2\").\n"
|
||||
"Falling back to \"KDTree\" nearest neighbor strategy (by default).")
|
||||
"Falling back to \"KDTree\" nearest neighbor strategy with Euclidean_L2 distance (by default).")
|
||||
.arg(descriptorBox->currentText())
|
||||
.arg(nnBox->currentText()));
|
||||
QString tmp = Settings::getNearestNeighbor_1Strategy();
|
||||
*tmp.begin() = '1'; // set index
|
||||
*tmp.begin() = '1'; // set KDTree
|
||||
Settings::setNearestNeighbor_1Strategy(tmp);
|
||||
tmp = Settings::getNearestNeighbor_2Distance_type();
|
||||
*tmp.begin() = '0'; // set EUCLIDEAN_L2
|
||||
Settings::setNearestNeighbor_2Distance_type(tmp);
|
||||
nnBox->blockSignals(true);
|
||||
this->updateParameter(Settings::kNearestNeighbor_1Strategy());
|
||||
this->updateParameter(Settings::kNearestNeighbor_2Distance_type());
|
||||
nnBox->blockSignals(false);
|
||||
if(sender() == nnBox)
|
||||
{
|
||||
|
||||
@ -176,8 +176,9 @@ class Settings
|
||||
PARAMETER(General, multiDetection, bool, false, "Multiple detection of the same object.");
|
||||
PARAMETER(General, multiDetectionRadius, int, 30, "Ignore detection of the same object in X pixels radius of the previous detections.");
|
||||
PARAMETER(General, port, int, 0, "Port on objects detected are published. If port=0, a port is chosen automatically.")
|
||||
PARAMETER(General, incrementalVocabulary, bool, false, "The vocabulary is created incrementally. When new objects are added, their descriptors are compared to those already in vocabulary to find if the visual word already exist or not. \"NearestNeighbor/nndrRatio\" is used to compare descriptors.");
|
||||
PARAMETER(General, autoScroll, bool, true, "Auto scroll to detected object in Objects panel.");
|
||||
PARAMETER(General, vocabularyIncremental, bool, false, "The vocabulary is created incrementally. When new objects are added, their descriptors are compared to those already in vocabulary to find if the visual word already exist or not. \"NearestNeighbor/nndrRatio\" is used to compare descriptors.");
|
||||
PARAMETER(General, vocabularyUpdateMinWords, int, 2000, "When the vocabulary is incremental (see \"General/vocabularyIncremental\"), after X words added to vocabulary, the internal index is updated with new words. This parameter lets avoiding to reconstruct the whole nearest neighbor index after each time descriptors of an object are added to vocabulary.");
|
||||
|
||||
PARAMETER(Homography, homographyComputed, bool, true, "Compute homography? On ROS, this is required to publish objects detected.");
|
||||
PARAMETER(Homography, method, QString, "1:LMEDS;RANSAC", "Type of the robust estimation algorithm: least-median algorithm or RANSAC algorithm.");
|
||||
|
||||
@ -20,8 +20,10 @@ Vocabulary::~Vocabulary()
|
||||
|
||||
void Vocabulary::clear()
|
||||
{
|
||||
descriptors_ = cv::Mat();
|
||||
indexedDescriptors_ = cv::Mat();
|
||||
notIndexedDescriptors_ = cv::Mat();
|
||||
wordToObjects_.clear();
|
||||
notIndexedWordIds_.clear();
|
||||
}
|
||||
|
||||
QMultiMap<int, int> Vocabulary::addWords(const cv::Mat & descriptors, int objectIndex, bool incremental)
|
||||
@ -35,43 +37,84 @@ QMultiMap<int, int> Vocabulary::addWords(const cv::Mat & descriptors, int object
|
||||
if(incremental)
|
||||
{
|
||||
int k = 2;
|
||||
cv::Mat results(descriptors.rows, k, CV_32SC1); // results index
|
||||
cv::Mat dists(descriptors.rows, k, CV_32FC1); // Distance results are CV_32FC1
|
||||
cv::Mat results;
|
||||
cv::Mat dists;
|
||||
|
||||
bool globalSearch = false;
|
||||
if(!descriptors_.empty() && descriptors_.rows >= (int)k)
|
||||
if(!indexedDescriptors_.empty() && indexedDescriptors_.rows >= (int)k)
|
||||
{
|
||||
Q_ASSERT(indexedDescriptors_.type() == descriptors.type() && indexedDescriptors_.cols == descriptors.cols);
|
||||
flannIndex_.knnSearch(descriptors, results, dists, k, Settings::getFlannSearchParams() );
|
||||
|
||||
if( dists.type() == CV_32S )
|
||||
{
|
||||
cv::Mat temp;
|
||||
dists.convertTo(temp, CV_32F);
|
||||
dists = temp;
|
||||
}
|
||||
|
||||
globalSearch = true;
|
||||
}
|
||||
|
||||
QVector<int> newWordsId; // index global
|
||||
cv::Mat newWords;
|
||||
QVector<int> newWordIds = notIndexedWordIds_; // index global
|
||||
cv::Mat newWords = notIndexedDescriptors_;
|
||||
int matches = 0;
|
||||
for(int i = 0; i < descriptors.rows; ++i)
|
||||
{
|
||||
QMap<float, int> fullResults; // nearest descriptors sorted by distance
|
||||
if(newWords.rows)
|
||||
{
|
||||
Q_ASSERT(newWords.type() == descriptors.type() && newWords.cols == descriptors.cols);
|
||||
|
||||
// Check if this descriptor matches with a word not already added to the vocabulary
|
||||
cv::flann::Index tmpIndex;
|
||||
cv::flann::IndexParams * params;
|
||||
// Do linear search only
|
||||
cv::Mat tmpResults;
|
||||
cv::Mat tmpDists;
|
||||
if(descriptors.type()==CV_8U)
|
||||
{
|
||||
params = Settings::createFlannIndexParams(); // should be LSH
|
||||
//normType – One of NORM_L1, NORM_L2, NORM_HAMMING, NORM_HAMMING2. L1 and L2 norms are
|
||||
// preferable choices for SIFT and SURF descriptors, NORM_HAMMING should be
|
||||
// used with ORB, BRISK and BRIEF, NORM_HAMMING2 should be used with ORB
|
||||
// when WTA_K==3 or 4 (see ORB::ORB constructor description).
|
||||
int normType = cv::NORM_HAMMING;
|
||||
if(Settings::currentDescriptorType().compare("ORB") &&
|
||||
(Settings::getFeature2D_ORB_WTA_K()==3 || Settings::getFeature2D_ORB_WTA_K()==4))
|
||||
{
|
||||
normType = cv::NORM_HAMMING2;
|
||||
}
|
||||
|
||||
cv::batchDistance( descriptors.row(i),
|
||||
newWords,
|
||||
tmpDists,
|
||||
CV_32S,
|
||||
tmpResults,
|
||||
normType,
|
||||
newWords.rows>=k?k:1,
|
||||
cv::Mat(),
|
||||
0,
|
||||
false);
|
||||
}
|
||||
else
|
||||
{
|
||||
params = new cv::flann::LinearIndexParams(); // faster
|
||||
cv::flann::Index tmpIndex;
|
||||
tmpIndex.build(newWords, cv::flann::LinearIndexParams(), Settings::getFlannDistanceType());
|
||||
tmpIndex.knnSearch(descriptors.row(i), tmpResults, tmpDists, newWords.rows>1?k:1, Settings::getFlannSearchParams());
|
||||
}
|
||||
tmpIndex.build(newWords, *params, Settings::getFlannDistanceType());
|
||||
delete params;
|
||||
cv::Mat tmpResults(1, newWords.rows>1?k:1, CV_32SC1); // results index
|
||||
cv::Mat tmpDists(1, newWords.rows>1?k:1, CV_32FC1); // Distance results are CV_32FC1
|
||||
tmpIndex.knnSearch(descriptors.row(i), tmpResults, tmpDists, newWords.rows>1?k:1, Settings::getFlannSearchParams());
|
||||
for(int j = 0; j < (newWords.rows>1?k:1); ++j)
|
||||
|
||||
if( tmpDists.type() == CV_32S )
|
||||
{
|
||||
fullResults.insert(tmpDists.at<float>(0,j), newWordsId.at(tmpResults.at<int>(0,j)));
|
||||
cv::Mat temp;
|
||||
tmpDists.convertTo(temp, CV_32F);
|
||||
tmpDists = temp;
|
||||
}
|
||||
|
||||
for(int j = 0; j < (newWords.rows>=k?k:1); ++j)
|
||||
{
|
||||
if(tmpResults.at<int>(0,j) >= 0)
|
||||
{
|
||||
//printf("local i=%d, j=%d, tmpDist=%f tmpResult=%d\n", i ,j, tmpDists.at<float>(0,j), tmpResults.at<int>(0,j));
|
||||
fullResults.insert(tmpDists.at<float>(0,j), newWordIds.at(tmpResults.at<int>(0,j)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -79,7 +122,11 @@ QMultiMap<int, int> Vocabulary::addWords(const cv::Mat & descriptors, int object
|
||||
{
|
||||
for(int j=0; j<k; ++j)
|
||||
{
|
||||
fullResults.insert(dists.at<float>(i,j), results.at<int>(i,j));
|
||||
if(results.at<int>(i,j) >= 0)
|
||||
{
|
||||
//printf("global i=%d, j=%d, dist=%f\n", i ,j, dists.at<float>(i,j));
|
||||
fullResults.insert(dists.at<float>(i,j), results.at<int>(i,j));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -107,49 +154,39 @@ QMultiMap<int, int> Vocabulary::addWords(const cv::Mat & descriptors, int object
|
||||
}
|
||||
cv::Mat dest(tmp, cv::Range(newWords.rows, newWords.rows+1));
|
||||
descriptors.row(i).copyTo(dest);
|
||||
newWordsId.push_back(descriptors_.rows + newWords.rows);
|
||||
newWordIds.push_back(indexedDescriptors_.rows + newWords.rows);
|
||||
newWords = tmp;
|
||||
words.insert(newWordsId.back(), i);
|
||||
wordToObjects_.insert(newWordsId.back(), objectIndex);
|
||||
words.insert(newWordIds.back(), i);
|
||||
wordToObjects_.insert(newWordIds.back(), objectIndex);
|
||||
}
|
||||
}
|
||||
//printf("matches = %d\n", matches);
|
||||
|
||||
//concatenate new words
|
||||
if(newWords.rows)
|
||||
{
|
||||
cv::Mat tmp(descriptors_.rows+newWords.rows, descriptors.cols, descriptors.type());
|
||||
if(descriptors_.rows)
|
||||
{
|
||||
cv::Mat dest(tmp, cv::Range(0, descriptors_.rows));
|
||||
descriptors_.copyTo(dest);
|
||||
}
|
||||
cv::Mat dest(tmp, cv::Range(descriptors_.rows, descriptors_.rows+newWords.rows));
|
||||
newWords.copyTo(dest);
|
||||
descriptors_ = tmp;
|
||||
notIndexedWordIds_ = newWordIds;
|
||||
notIndexedDescriptors_ = newWords;
|
||||
}
|
||||
|
||||
//update
|
||||
this->update();
|
||||
}
|
||||
else
|
||||
{
|
||||
for(int i = 0; i < descriptors.rows; ++i)
|
||||
{
|
||||
wordToObjects_.insert(descriptors_.rows+i, objectIndex);
|
||||
words.insert(descriptors_.rows+i, i);
|
||||
wordToObjects_.insert(indexedDescriptors_.rows + notIndexedDescriptors_.rows+i, objectIndex);
|
||||
words.insert(indexedDescriptors_.rows + notIndexedDescriptors_.rows+i, i);
|
||||
notIndexedWordIds_.push_back(indexedDescriptors_.rows + notIndexedDescriptors_.rows+i);
|
||||
}
|
||||
|
||||
//just concatenate descriptors
|
||||
cv::Mat tmp(descriptors_.rows+descriptors.rows, descriptors.cols, descriptors.type());
|
||||
if(descriptors_.rows)
|
||||
cv::Mat tmp(notIndexedDescriptors_.rows+descriptors.rows, descriptors.cols, descriptors.type());
|
||||
if(notIndexedDescriptors_.rows)
|
||||
{
|
||||
cv::Mat dest(tmp, cv::Range(0, descriptors_.rows));
|
||||
descriptors_.copyTo(dest);
|
||||
cv::Mat dest(tmp, cv::Range(0, notIndexedDescriptors_.rows));
|
||||
notIndexedDescriptors_.copyTo(dest);
|
||||
}
|
||||
cv::Mat dest(tmp, cv::Range(descriptors_.rows, descriptors_.rows+descriptors.rows));
|
||||
cv::Mat dest(tmp, cv::Range(notIndexedDescriptors_.rows, notIndexedDescriptors_.rows+descriptors.rows));
|
||||
descriptors.copyTo(dest);
|
||||
descriptors_ = tmp;
|
||||
notIndexedDescriptors_ = tmp;
|
||||
}
|
||||
|
||||
return words;
|
||||
@ -157,18 +194,46 @@ QMultiMap<int, int> Vocabulary::addWords(const cv::Mat & descriptors, int object
|
||||
|
||||
void Vocabulary::update()
|
||||
{
|
||||
if(!descriptors_.empty())
|
||||
if(!notIndexedDescriptors_.empty())
|
||||
{
|
||||
Q_ASSERT(indexedDescriptors_.cols == notIndexedDescriptors_.cols &&
|
||||
indexedDescriptors_.type() == notIndexedDescriptors_.type() );
|
||||
|
||||
//concatenate descriptors
|
||||
cv::Mat tmp(indexedDescriptors_.rows+notIndexedDescriptors_.rows, notIndexedDescriptors_.cols, notIndexedDescriptors_.type());
|
||||
cv::Mat dest(tmp, cv::Range(0, indexedDescriptors_.rows));
|
||||
indexedDescriptors_.copyTo(dest);
|
||||
dest = cv::Mat(tmp, cv::Range(indexedDescriptors_.rows, indexedDescriptors_.rows+notIndexedDescriptors_.rows));
|
||||
notIndexedDescriptors_.copyTo(dest);
|
||||
indexedDescriptors_ = tmp;
|
||||
|
||||
notIndexedDescriptors_ = cv::Mat();
|
||||
notIndexedWordIds_.clear();
|
||||
}
|
||||
|
||||
if(!indexedDescriptors_.empty())
|
||||
{
|
||||
cv::flann::IndexParams * params = Settings::createFlannIndexParams();
|
||||
flannIndex_.build(descriptors_, *params, Settings::getFlannDistanceType());
|
||||
flannIndex_.build(indexedDescriptors_, *params, Settings::getFlannDistanceType());
|
||||
delete params;
|
||||
}
|
||||
}
|
||||
|
||||
void Vocabulary::search(const cv::Mat & descriptors, cv::Mat & results, cv::Mat & dists, int k)
|
||||
{
|
||||
if(!descriptors_.empty())
|
||||
Q_ASSERT(notIndexedDescriptors_.empty() && notIndexedWordIds_.size() == 0);
|
||||
|
||||
if(!indexedDescriptors_.empty())
|
||||
{
|
||||
Q_ASSERT(descriptors.type() == indexedDescriptors_.type() && descriptors.cols == indexedDescriptors_.cols);
|
||||
|
||||
flannIndex_.knnSearch(descriptors, results, dists, k, Settings::getFlannSearchParams());
|
||||
|
||||
if( dists.type() == CV_32S )
|
||||
{
|
||||
cv::Mat temp;
|
||||
dists.convertTo(temp, CV_32F);
|
||||
dists = temp;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
#define VOCABULARY_H_
|
||||
|
||||
#include <QtCore/QMultiMap>
|
||||
#include <QtCore/QVector>
|
||||
#include <opencv2/opencv.hpp>
|
||||
|
||||
class Vocabulary {
|
||||
@ -20,13 +21,15 @@ public:
|
||||
QMultiMap<int, int> addWords(const cv::Mat & descriptors, int objectIndex, bool incremental);
|
||||
void update();
|
||||
void search(const cv::Mat & descriptors, cv::Mat & results, cv::Mat & dists, int k);
|
||||
int size() const {return descriptors_.rows;}
|
||||
int size() const {return indexedDescriptors_.rows + notIndexedDescriptors_.rows;}
|
||||
const QMultiMap<int, int> & wordToObjects() const {return wordToObjects_;}
|
||||
|
||||
private:
|
||||
cv::flann::Index flannIndex_;
|
||||
cv::Mat descriptors_;
|
||||
cv::Mat indexedDescriptors_;
|
||||
cv::Mat notIndexedDescriptors_;
|
||||
QMultiMap<int, int> wordToObjects_;
|
||||
QVector<int> notIndexedWordIds_;
|
||||
};
|
||||
|
||||
#endif /* VOCABULARY_H_ */
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user