From f1bba5b8fbe74dd3332c8a9bfaa411e57cb58712 Mon Sep 17 00:00:00 2001 From: matlabbe Date: Wed, 24 Jun 2015 18:07:52 -0400 Subject: [PATCH] Added wordID in shown keypoint info, improved fixed vocabulary behavior --- example/main.cpp | 6 +- include/find_object/FindObject.h | 2 +- include/find_object/MainWindow.h | 5 +- include/find_object/ObjWidget.h | 10 +- include/find_object/Settings.h | 3 +- src/AddObjectDialog.cpp | 8 +- src/FindObject.cpp | 218 +++++++++++++++++++------------ src/KeypointItem.cpp | 17 ++- src/KeypointItem.h | 7 +- src/MainWindow.cpp | 96 ++++++++++---- src/ObjWidget.cpp | 62 ++++++--- src/Vocabulary.h | 2 + 12 files changed, 288 insertions(+), 148 deletions(-) diff --git a/example/main.cpp b/example/main.cpp index 65e4494f..26b3e8f9 100644 --- a/example/main.cpp +++ b/example/main.cpp @@ -79,8 +79,6 @@ int main(int argc, char * argv[]) // GUI stuff QApplication app(argc, argv); - ObjWidget objWidget; - ObjWidget sceneWidget; time.start(); //Load as grayscale @@ -199,8 +197,8 @@ int main(int argc, char * argv[]) // PROCESS NEAREST NEIGHBOR RESULTS //////////////////////////// // Set gui data - objWidget.setData(objectKeypoints, cvtCvMat2QImage(objectImg)); - sceneWidget.setData(sceneKeypoints, cvtCvMat2QImage(sceneImg)); + ObjWidget objWidget(0, objectKeypoints, QMultiMap(), cvtCvMat2QImage(objectImg)); + ObjWidget sceneWidget(0, sceneKeypoints, QMultiMap(), cvtCvMat2QImage(sceneImg)); // Find correspondences by NNDR (Nearest Neighbor Distance Ratio) float nndrRatio = 0.8f; diff --git a/include/find_object/FindObject.h b/include/find_object/FindObject.h index 3c1ee352..27273cc6 100644 --- a/include/find_object/FindObject.h +++ b/include/find_object/FindObject.h @@ -80,7 +80,7 @@ public: void updateDetectorExtractor(); void updateObjects(const QList & ids = QList()); - void updateVocabulary(); + void updateVocabulary(const QList & ids = QList()); const QMap & objects() const {return objects_;} const Vocabulary * vocabulary() const {return vocabulary_;} diff --git a/include/find_object/MainWindow.h b/include/find_object/MainWindow.h index 5a882c3e..62dd41b2 100644 --- a/include/find_object/MainWindow.h +++ b/include/find_object/MainWindow.h @@ -104,7 +104,7 @@ private Q_SLOTS: void showHideControls(); void showObjectsFeatures(); void hideObjectsFeatures(); - void updateObjects(); + void updateObjects(const QList & ids = QList()); void notifyParametersChanged(const QStringList & param); void moveCameraFrame(int frame); void rectHovered(int objId); @@ -121,8 +121,7 @@ private: int addObjectFromFile(const QString & filePath); void showObject(find_object::ObjWidget * obj); void updateObjectSize(find_object::ObjWidget * obj); - void updateVocabulary(); - void updateObjects(const QList & ids); + void updateVocabulary(const QList & ids = QList()); private: Ui_mainWindow * ui_; diff --git a/include/find_object/ObjWidget.h b/include/find_object/ObjWidget.h index 24294779..8e133a42 100644 --- a/include/find_object/ObjWidget.h +++ b/include/find_object/ObjWidget.h @@ -53,14 +53,18 @@ class FINDOBJECT_EXP ObjWidget : public QWidget public: ObjWidget(QWidget * parent = 0); - ObjWidget(int id, const std::vector & keypoints, const QImage & image, QWidget * parent = 0); + ObjWidget(int id, const std::vector & keypoints, const QMultiMap & words, const QImage & image, QWidget * parent = 0); virtual ~ObjWidget(); void setId(int id); - void setData(const std::vector & keypoints, const QImage & image, const QRect & rect = QRect()); + void updateImage(const QImage & image); + void updateData(const std::vector & keypoints, const QMultiMap & words=QMultiMap()); + void updateWords(const QMultiMap & words); void setTextLabel(const QString & text); void resetKptsColor(); + void resetKptsWordID(); void setKptColor(int index, const QColor & color); + void setKptWordID(int index, int wordId); void setGraphicsViewMode(bool on); void setAutoScale(bool autoScale); void setSizedFeatures(bool on); @@ -75,6 +79,7 @@ public: int id() const {return id_;} const QColor & color() const {return color_;} const std::vector keypoints() const {return keypoints_;} + const QMap & words() const {return words_;} const QPixmap & pixmap() const {return pixmap_;} QColor defaultColor() const; bool isImageShown() const; @@ -110,6 +115,7 @@ private: private: int id_; std::vector keypoints_; + QMap words_; // QPixmap pixmap_; QRect rect_; QList keypointItems_; diff --git a/include/find_object/Settings.h b/include/find_object/Settings.h index f0579ad7..eae0d7ac 100644 --- a/include/find_object/Settings.h +++ b/include/find_object/Settings.h @@ -275,7 +275,8 @@ class FINDOBJECT_EXP Settings PARAMETER(General, multiDetectionRadius, int, 30, "Ignore detection of the same object in X pixels radius of the previous detections."); PARAMETER(General, port, int, 0, "Port on objects detected are published. If port=0, a port is chosen automatically.") PARAMETER(General, autoScroll, bool, true, "Auto scroll to detected object in Objects panel."); - PARAMETER(General, vocabularyIncremental, bool, false, "The vocabulary is created incrementally. When new objects are added, their descriptors are compared to those already in vocabulary to find if the visual word already exist or not. \"NearestNeighbor/nndrRatio\" is used to compare descriptors."); + PARAMETER(General, vocabularyFixed, bool, false, "If the vocabulary is fixed, no new words will be added to it when adding new objects."); + PARAMETER(General, vocabularyIncremental, bool, false, "The vocabulary is created incrementally. When new objects are added, their descriptors are compared to those already in vocabulary to find if the visual word already exist or not. \"NearestNeighbor/nndrRatio\" and \"NearestNeighbor/minDistance\" are used to compare descriptors."); PARAMETER(General, vocabularyUpdateMinWords, int, 2000, "When the vocabulary is incremental (see \"General/vocabularyIncremental\"), after X words added to vocabulary, the internal index is updated with new words. This parameter lets avoiding to reconstruct the whole nearest neighbor index after each time descriptors of an object are added to vocabulary. 0 means no incremental update."); PARAMETER(General, sendNoObjDetectedEvents, bool, true, "When there are no objects detected, send an empty object detection event."); PARAMETER(General, autoPauseOnDetection, bool, false, "Auto pause the camera when an object is detected."); diff --git a/src/AddObjectDialog.cpp b/src/AddObjectDialog.cpp index 505f8c7c..3580cf59 100644 --- a/src/AddObjectDialog.cpp +++ b/src/AddObjectDialog.cpp @@ -316,7 +316,8 @@ void AddObjectDialog::setState(int state) selectedKeypoints.clear(); detector_->detect(imgRoi, selectedKeypoints); } - ui_->objectView->setData(selectedKeypoints, cvtCvMat2QImage(imgRoi.clone())); + ui_->objectView->updateImage(cvtCvMat2QImage(imgRoi.clone())); + ui_->objectView->updateData(selectedKeypoints, QMultiMap()); ui_->objectView->setMinimumSize(roi_.width, roi_.height); ui_->objectView->update(); ui_->pushButton_next->setEnabled(true); @@ -359,7 +360,7 @@ void AddObjectDialog::setState(int state) } objSignature_ = new ObjSignature(0, imgRoi.clone(), ""); objSignature_->setData(keypoints, descriptors); - objWidget_ = new ObjWidget(0, keypoints, cvtCvMat2QImage(imgRoi.clone())); + objWidget_ = new ObjWidget(0, keypoints, QMultiMap(), cvtCvMat2QImage(imgRoi.clone())); this->accept(); } @@ -385,7 +386,8 @@ void AddObjectDialog::update(const cv::Mat & image) std::vector keypoints; detector_->detect(cameraImage_, keypoints); - ui_->cameraView->setData(keypoints, cvtCvMat2QImage(cameraImage_)); + ui_->cameraView->updateImage(cvtCvMat2QImage(cameraImage_)); + ui_->cameraView->updateData(keypoints, QMultiMap()); ui_->cameraView->update(); } else diff --git a/src/FindObject.cpp b/src/FindObject.cpp index 352a2345..f3742549 100644 --- a/src/FindObject.cpp +++ b/src/FindObject.cpp @@ -28,6 +28,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "find_object/FindObject.h" #include "find_object/Settings.h" #include "find_object/utilite/ULogger.h" +#include "utilite/UConversion.h" #include "ObjSignature.h" #include "utilite/UDirectory.h" @@ -144,12 +145,12 @@ bool FindObject::saveSession(const QString & path) int FindObject::loadObjects(const QString & dirPath, bool recursive) { - int loadedObjects = 0; QString formats = Settings::getGeneral_imageFormats().remove('*').remove('.'); QStringList paths; paths.append(dirPath); + QList idsLoaded; while(paths.size()) { QString currentDir = paths.front(); @@ -159,9 +160,10 @@ int FindObject::loadObjects(const QString & dirPath, bool recursive) const std::list & names = dir.getFileNames(); // sorted in natural order for(std::list::const_iterator iter=names.begin(); iter!=names.end(); ++iter) { - if(this->addObject((currentDir.toStdString()+dir.separator()+*iter).c_str())) + const ObjSignature * s = this->addObject((currentDir.toStdString()+dir.separator()+*iter).c_str()); + if(s) { - ++loadedObjects; + idsLoaded.push_back(s->id()); } } } @@ -179,13 +181,13 @@ int FindObject::loadObjects(const QString & dirPath, bool recursive) } } - if(loadedObjects) + if(idsLoaded.size()) { - this->updateObjects(); - this->updateVocabulary(); + this->updateObjects(idsLoaded); + this->updateVocabulary(idsLoaded); } - return loadedObjects; + return idsLoaded.size(); } const ObjSignature * FindObject::addObject(const QString & filePath) @@ -279,7 +281,7 @@ void FindObject::addObjectAndUpdate(const cv::Mat & image, int id, const QString QList ids; ids.push_back(s->id()); updateObjects(ids); - updateVocabulary(); + updateVocabulary(ids); } } @@ -739,14 +741,38 @@ void FindObject::clearVocabulary() vocabulary_->clear(); } -void FindObject::updateVocabulary() +void FindObject::updateVocabulary(const QList & ids) { - clearVocabulary(); int count = 0; int dim = -1; int type = -1; + QList objectsList; + if(ids.size()) + { + for(int i=0; isize()) + { + dim = vocabulary_->dim(); + type = vocabulary_->type(); + } + } + else + { + clearVocabulary(); + objectsList = objects_.values(); + } + // Get the total size and verify descriptors - QList objectsList = objects_.values(); for(int i=0; idescriptors().empty()) @@ -774,94 +800,119 @@ void FindObject::updateVocabulary() { UINFO("Updating global descriptors matrix: Objects=%d, total descriptors=%d, dim=%d, type=%d", (int)objects_.size(), count, dim, type); - if(Settings::getGeneral_invertedSearch() || Settings::getGeneral_threads() == 1) + if(!Settings::getGeneral_invertedSearch()) { - // If only one thread, put all descriptors in the same cv::Mat - objectsDescriptors_.insert(0, cv::Mat(count, dim, type)); - int row = 0; - for(int i=0; idescriptors().rows) + // If only one thread, put all descriptors in the same cv::Mat + int row = 0; + bool vocabularyEmpty = objectsDescriptors_.size() == 0; + if(vocabularyEmpty) { - cv::Mat dest(objectsDescriptors_.begin().value(), cv::Range(row, row+objectsList.at(i)->descriptors().rows)); - objectsList.at(i)->descriptors().copyTo(dest); - row += objectsList.at(i)->descriptors().rows; - // dataRange contains the upper_bound for each - // object (the last descriptors position in the - // global object descriptors matrix) + UASSERT(objectsDescriptors_.size() == 0); + objectsDescriptors_.insert(0, cv::Mat(count, dim, type)); + } + else + { + row = objectsDescriptors_.begin().value().rows; + } + for(int i=0; isetWords(QMultiMap()); if(objectsList.at(i)->descriptors().rows) { - dataRange_.insert(row-1, objectsList.at(i)->id()); + if(vocabularyEmpty) + { + cv::Mat dest(objectsDescriptors_.begin().value(), cv::Range(row, row+objectsList.at(i)->descriptors().rows)); + objectsList.at(i)->descriptors().copyTo(dest); + } + else + { + UASSERT_MSG(objectsDescriptors_.begin().value().cols == objectsList.at(i)->descriptors().cols, + uFormat("%d vs %d", objectsDescriptors_.begin().value().cols, objectsList.at(i)->descriptors().cols).c_str()); + UASSERT(objectsDescriptors_.begin().value().type() == objectsList.at(i)->descriptors().type()); + objectsDescriptors_.begin().value().push_back(objectsList.at(i)->descriptors()); + } + + row += objectsList.at(i)->descriptors().rows; + // dataRange contains the upper_bound for each + // object (the last descriptors position in the + // global object descriptors matrix) + if(objectsList.at(i)->descriptors().rows) + { + dataRange_.insert(row-1, objectsList.at(i)->id()); + } } } } - - if(Settings::getGeneral_invertedSearch()) + else { - sessionModified_ = true; - QTime time; - time.start(); - bool incremental = Settings::getGeneral_vocabularyIncremental() && !Settings::getGeneral_vocabularyFixed(); - if(incremental) - { - UINFO("Creating incremental vocabulary..."); - } - else if(Settings::getGeneral_vocabularyFixed()) - { - UINFO("Updating vocabulary correspondences only (vocabulary is fixed)..."); - } - else - { - UINFO("Creating vocabulary..."); - } - QTime localTime; - localTime.start(); - int updateVocabularyMinWords = Settings::getGeneral_vocabularyUpdateMinWords(); - int addedWords = 0; for(int i=0; i words = vocabulary_->addWords(objectsList[i]->descriptors(), objectsList.at(i)->id()); - objectsList[i]->setWords(words); - addedWords += words.uniqueKeys().size(); - bool updated = false; - if(incremental && addedWords && addedWords >= updateVocabularyMinWords) - { - vocabulary_->update(); - addedWords = 0; - updated = true; - } - UINFO("Object %d, %d words from %d descriptors (%d words, %d ms) %s", - objectsList[i]->id(), - words.uniqueKeys().size(), - objectsList[i]->descriptors().rows, - vocabulary_->size(), - localTime.restart(), - updated?"updated":""); - } - if(addedWords && !Settings::getGeneral_vocabularyFixed()) - { - vocabulary_->update(); - } - - if(incremental) - { - UINFO("Creating incremental vocabulary... done! size=%d (%d ms)", vocabulary_->size(), time.elapsed()); - } - else if(Settings::getGeneral_vocabularyFixed()) - { - UINFO("Updating vocabulary correspondences only (vocabulary is fixed)... done! size=%d (%d ms)", time.elapsed()); - } - else - { - UINFO("Creating vocabulary... done! size=%d (%d ms)", vocabulary_->size(), time.elapsed()); + objectsList[i]->setWords(QMultiMap()); + objectsDescriptors_.insert(objectsList.at(i)->id(), objectsList.at(i)->descriptors()); } } } else { + // Inverted index on (vocabulary) + sessionModified_ = true; + QTime time; + time.start(); + bool incremental = Settings::getGeneral_vocabularyIncremental() && !Settings::getGeneral_vocabularyFixed(); + if(incremental) + { + UINFO("Creating incremental vocabulary..."); + } + else if(Settings::getGeneral_vocabularyFixed()) + { + UINFO("Updating vocabulary correspondences only (vocabulary is fixed)..."); + } + else + { + UINFO("Creating vocabulary..."); + } + QTime localTime; + localTime.start(); + int updateVocabularyMinWords = Settings::getGeneral_vocabularyUpdateMinWords(); + int addedWords = 0; for(int i=0; iid(), objectsList.at(i)->descriptors()); + QMultiMap words = vocabulary_->addWords(objectsList[i]->descriptors(), objectsList.at(i)->id()); + objectsList[i]->setWords(words); + addedWords += words.uniqueKeys().size(); + bool updated = false; + if(incremental && addedWords && addedWords >= updateVocabularyMinWords) + { + vocabulary_->update(); + addedWords = 0; + updated = true; + } + UINFO("Object %d, %d words from %d descriptors (%d words, %d ms) %s", + objectsList[i]->id(), + words.uniqueKeys().size(), + objectsList[i]->descriptors().rows, + vocabulary_->size(), + localTime.restart(), + updated?"updated":""); + } + if(addedWords && !Settings::getGeneral_vocabularyFixed()) + { + vocabulary_->update(); + } + + if(incremental) + { + UINFO("Creating incremental vocabulary... done! size=%d (%d ms)", vocabulary_->size(), time.elapsed()); + } + else if(Settings::getGeneral_vocabularyFixed()) + { + UINFO("Updating vocabulary correspondences only (vocabulary is fixed)... done! size=%d (%d ms)", vocabulary_->size(), time.elapsed()); + } + else + { + UINFO("Creating vocabulary... done! size=%d (%d ms)", vocabulary_->size(), time.elapsed()); } } } @@ -1202,6 +1253,7 @@ bool FindObject::detect(const cv::Mat & image, find_object::DetectionInfo & info words = vocabulary_->addWords(info.sceneDescriptors_, -1); vocabulary_->update(); info.timeStamps_.insert(DetectionInfo::kTimeIndexing, time.restart()); + info.sceneWords_ = words; } for(QMap::iterator iter=objects_.begin(); iter!=objects_.end(); ++iter) @@ -1274,9 +1326,10 @@ bool FindObject::detect(const cv::Mat & image, find_object::DetectionInfo & info if(matched) { + int wordId = results.at(i,0); if(Settings::getGeneral_invertedSearch()) { - int wordId = results.at(i,0); + info.sceneWords_.insertMulti(wordId, i); QList objIds = vocabulary_->wordToObjects().values(wordId); for(int j=0; j(i,0); if(words.count(wordId) == 1) { info.matches_.find(objectId).value().insert(objectDescriptorIndex, words.value(wordId)); diff --git a/src/KeypointItem.cpp b/src/KeypointItem.cpp index 8f2cff8b..c044fd31 100644 --- a/src/KeypointItem.cpp +++ b/src/KeypointItem.cpp @@ -33,11 +33,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace find_object { -KeypointItem::KeypointItem(int id, qreal x, qreal y, int r, const QString & info, const QColor & color, QGraphicsItem * parent) : +KeypointItem::KeypointItem(int id, qreal x, qreal y, int r, const cv::KeyPoint & kpt, int wordID, const QColor & color, QGraphicsItem * parent) : QGraphicsEllipseItem(x, y, r, r, parent), - info_(info), placeHolder_(0), - id_(id) + id_(id), + kpt_(kpt), + wordID_(wordID) { this->setPen(QPen(color)); this->setBrush(QBrush(color)); @@ -70,13 +71,21 @@ void KeypointItem::showDescription() { if(!placeHolder_) { + QString info = QString( "Keypoint = %1\n" + "Word = %2\n" + "Response = %3\n" + "Angle = %4\n" + "X = %5\n" + "Y = %6\n" + "Size = %7").arg(id_).arg(wordID_).arg(kpt_.response).arg(kpt_.angle).arg(kpt_.pt.x).arg(kpt_.pt.y).arg(kpt_.size); + placeHolder_ = new QGraphicsRectItem(); placeHolder_->setVisible(false); this->scene()->addItem(placeHolder_); placeHolder_->setBrush(QBrush(QColor ( 0, 0, 0, 170 ))); // Black transparent background QGraphicsTextItem * text = new QGraphicsTextItem(placeHolder_); text->setDefaultTextColor(this->pen().color().rgb()); - text->setPlainText(info_); + text->setPlainText(info); placeHolder_->setRect(text->boundingRect()); } diff --git a/src/KeypointItem.h b/src/KeypointItem.h index e6c886bd..1f7e3cef 100644 --- a/src/KeypointItem.h +++ b/src/KeypointItem.h @@ -32,16 +32,18 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include namespace find_object { class KeypointItem : public QGraphicsEllipseItem { public: - KeypointItem(int id, qreal x, qreal y, int r, const QString & info, const QColor & color = Qt::green, QGraphicsItem * parent = 0); + KeypointItem(int id, qreal x, qreal y, int r, const cv::KeyPoint & kpt, int wordID = -1, const QColor & color = Qt::green, QGraphicsItem * parent = 0); virtual ~KeypointItem(); void setColor(const QColor & color); + void setWordID(int id) {wordID_ = id;} int id() const {return id_;} protected: @@ -55,9 +57,10 @@ private: void hideDescription(); private: - QString info_; QGraphicsRectItem * placeHolder_; int id_; + cv::KeyPoint kpt_; + int wordID_; }; } // namespace find_object diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index edaf8387..b2ab6bd8 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -183,7 +183,7 @@ MainWindow::MainWindow(FindObject * findObject, Camera * camera, QWidget * paren //buttons connect(ui_->pushButton_restoreDefaults, SIGNAL(clicked()), ui_->toolBox, SLOT(resetCurrentPage())); - connect(ui_->pushButton_updateObjects, SIGNAL(clicked()), this, SLOT(updateObjects())); + connect(ui_->pushButton_updateObjects, SIGNAL(clicked()), this, SLOT(updateObjects(const QList &))); connect(ui_->horizontalSlider_objectsSize, SIGNAL(valueChanged(int)), this, SLOT(updateObjectsSize())); ui_->actionStop_camera->setEnabled(false); @@ -247,7 +247,7 @@ MainWindow::MainWindow(FindObject * findObject, Camera * camera, QWidget * paren iter!=findObject_->objects().constEnd(); ++iter) { - ObjWidget * obj = new ObjWidget(iter.key(), iter.value()->keypoints(), cvtCvMat2QImage(iter.value()->image())); + ObjWidget * obj = new ObjWidget(iter.key(), iter.value()->keypoints(), iter.value()->words(), cvtCvMat2QImage(iter.value()->image())); objWidgets_.insert(obj->id(), obj); this->showObject(obj); } @@ -366,7 +366,7 @@ void MainWindow::loadSession() { if(iter.value()) { - ObjWidget * obj = new ObjWidget(iter.key(), iter.value()->keypoints(), cvtCvMat2QImage(iter.value()->image())); + ObjWidget * obj = new ObjWidget(iter.key(), iter.value()->keypoints(), iter.value()->words(), cvtCvMat2QImage(iter.value()->image())); objWidgets_.insert(obj->id(), obj); ui_->actionSave_objects->setEnabled(true); ui_->actionSave_session->setEnabled(true); @@ -714,7 +714,9 @@ void MainWindow::addObjectFromScene() ui_->actionSave_objects->setEnabled(true); ui_->actionSave_session->setEnabled(true); showObject(obj); - updateVocabulary(); + QList ids; + ids.push_back(obj->id()); + updateVocabulary(ids); objectsModified_ = true; } if(resumeCamera || sceneImage_.empty()) @@ -764,7 +766,7 @@ int MainWindow::addObjectFromFile(const QString & filePath) const ObjSignature * s = findObject_->addObject(filePath); if(s) { - ObjWidget * obj = new ObjWidget(s->id(), std::vector(), cvtCvMat2QImage(s->image())); + ObjWidget * obj = new ObjWidget(s->id(), std::vector(), QMultiMap(), cvtCvMat2QImage(s->image())); objWidgets_.insert(obj->id(), obj); ui_->actionSave_objects->setEnabled(true); ui_->actionSave_session->setEnabled(true); @@ -787,7 +789,7 @@ void MainWindow::addObjectFromTcp(const cv::Mat & image, int id, const QString & const ObjSignature * s = findObject_->addObject(image, id, filePath); if(s) { - ObjWidget * obj = new ObjWidget(s->id(), std::vector(), cvtCvMat2QImage(s->image())); + ObjWidget * obj = new ObjWidget(s->id(), std::vector(), QMultiMap(), cvtCvMat2QImage(s->image())); objWidgets_.insert(obj->id(), obj); ui_->actionSave_objects->setEnabled(true); ui_->actionSave_session->setEnabled(true); @@ -977,21 +979,24 @@ void MainWindow::showObject(ObjWidget * obj) void MainWindow::updateObjects(const QList & ids) { - if(ids.size()) + if(objWidgets_.size()) { - this->statusBar()->showMessage(tr("Updating %1 objects...").arg(ids.size())); + this->statusBar()->showMessage(tr("Updating %1 objects...").arg(ids.size()==0?objWidgets_.size():ids.size())); findObject_->updateObjects(ids); - updateVocabulary(); + QList idsTmp = ids; + if(idsTmp.size() == 0) + { + idsTmp = objWidgets_.keys(); + } QList signatures = findObject_->objects().values(); for(int i=0; iid())) + if(idsTmp.contains(signatures[i]->id())) { - QImage qtImage = cvtCvMat2QImage(signatures[i]->image()); - objWidgets_.value(signatures[i]->id())->setData(signatures[i]->keypoints(), qtImage, signatures[i]->rect()); + objWidgets_.value(signatures[i]->id())->updateData(signatures[i]->keypoints()); //update object labels QLabel * title = qFindChild(this, QString("%1title").arg(signatures[i]->id())); @@ -999,6 +1004,8 @@ void MainWindow::updateObjects(const QList & ids) } } + updateVocabulary(ids); + if(!camera_->isRunning() && !sceneImage_.empty()) { this->update(sceneImage_); @@ -1007,23 +1014,34 @@ void MainWindow::updateObjects(const QList & ids) } } -void MainWindow::updateObjects() -{ - updateObjects(objWidgets_.keys()); -} - -void MainWindow::updateVocabulary() +void MainWindow::updateVocabulary(const QList & ids) { this->statusBar()->showMessage(tr("Updating vocabulary...")); QTime time; time.start(); - findObject_->updateVocabulary(); + findObject_->updateVocabulary(ids); - if(findObject_->vocabulary()->size()) + QList idsTmp = ids; + if(idsTmp.size() == 0) { - ui_->label_timeIndexing->setNum(time.elapsed()); - ui_->label_vocabularySize->setNum(findObject_->vocabulary()->size()); + idsTmp = objWidgets_.keys(); + } + QList signatures = findObject_->objects().values(); + for(int i=0; iid())) + { + objWidgets_.value(signatures[i]->id())->updateWords(signatures[i]->words()); + } + } + + ui_->label_timeIndexing->setNum(time.elapsed()); + ui_->label_vocabularySize->setNum(findObject_->vocabulary()->size()); + if(ids.size() && findObject_->vocabulary()->size() == 0 && Settings::getGeneral_vocabularyFixed() && Settings::getGeneral_invertedSearch()) + { + QMessageBox::warning(this, tr("Vocabulary update"), tr("\"General/VocabularyFixed=true\" and the " + "vocabulary is empty. New features cannot be matched to any words in the vocabulary.")); } lastObjectsUpdateParameters_ = Settings::getParameters(); this->statusBar()->clearMessage(); @@ -1172,6 +1190,10 @@ void MainWindow::update(const cv::Mat & image) for(QMap::iterator iter=objWidgets_.begin(); iter!=objWidgets_.end(); ++iter) { iter.value()->resetKptsColor(); + if(!Settings::getGeneral_invertedSearch()) + { + iter.value()->resetKptsWordID(); + } } QTime guiRefreshTime; @@ -1184,7 +1206,8 @@ void MainWindow::update(const cv::Mat & image) ui_->label_timeSkewAffine->setNum(info.timeStamps_.value(DetectionInfo::kTimeSkewAffine, 0)); ui_->label_timeExtraction->setNum(info.timeStamps_.value(DetectionInfo::kTimeDescriptorExtraction, 0)); ui_->label_timeSubPix->setNum(info.timeStamps_.value(DetectionInfo::kTimeSubPixelRefining, 0)); - ui_->imageView_source->setData(info.sceneKeypoints_, cvtCvMat2QImage(sceneImage_)); + ui_->imageView_source->updateImage(cvtCvMat2QImage(sceneImage_)); + ui_->imageView_source->updateData(info.sceneKeypoints_, info.sceneWords_); if(!findObject_->vocabulary()->size()) { ui_->label_timeIndexing->setNum(info.timeStamps_.value(DetectionInfo::kTimeIndexing, 0)); @@ -1221,6 +1244,10 @@ void MainWindow::update(const cv::Mat & image) { obj->setKptColor(iter.key(), obj->color()); ui_->imageView_source->setKptColor(iter.value(), obj->color()); + if(!Settings::getGeneral_invertedSearch()) + { + obj->setKptWordID(iter.key(), ui_->imageView_source->words().value(iter.value(), -1)); + } } } else if(!info.objDetected_.contains(id)) @@ -1263,6 +1290,8 @@ void MainWindow::update(const cv::Mat & image) } // Add homography rectangles when homographies are computed + int maxHomographyScoreId = -1; + int maxHomographyScore = 0; QMultiMap >::const_iterator inliersIter = info.objDetectedInliers_.constBegin(); QMultiMap >::const_iterator outliersIter = info.objDetectedOutliers_.constBegin(); for(QMultiMap::iterator iter = info.objDetected_.begin(); @@ -1270,6 +1299,13 @@ void MainWindow::update(const cv::Mat & image) ++iter, ++inliersIter, ++outliersIter) { int id = iter.key(); + + if(maxHomographyScoreId == -1 || maxHomographyScore < inliersIter.value().size()) + { + maxHomographyScoreId = id; + maxHomographyScore = inliersIter.value().size(); + } + ObjWidget * obj = objWidgets_.value(id); UASSERT(obj != 0); @@ -1294,6 +1330,10 @@ void MainWindow::update(const cv::Mat & image) { obj->setKptColor(iter.key(), obj->color()); ui_->imageView_source->setKptColor(iter.value(), obj->color()); + if(!Settings::getGeneral_invertedSearch()) + { + obj->setKptWordID(iter.key(), ui_->imageView_source->words().value(iter.value(), -1)); + } } QLabel * label = ui_->dockWidget_objects->findChild(QString("%1detection").arg(id)); @@ -1339,9 +1379,9 @@ void MainWindow::update(const cv::Mat & image) ui_->label_maxMatchedDistance->setNum(info.maxMatchedDistance_); //Scroll objects slider to the best score - if(maxScoreId>=0 && Settings::getGeneral_autoScroll()) + if((maxScoreId>=0 || maxHomographyScoreId>=0) && Settings::getGeneral_autoScroll()) { - QLabel * label = ui_->dockWidget_objects->findChild(QString("%1title").arg(maxScoreId)); + QLabel * label = ui_->dockWidget_objects->findChild(QString("%1title").arg(maxHomographyScoreId>=0?maxHomographyScoreId:maxScoreId)); if(label) { ui_->objects_area->verticalScrollBar()->setValue(label->pos().y()); @@ -1385,7 +1425,8 @@ void MainWindow::update(const cv::Mat & image) ui_->label_timeSkewAffine->setNum(info.timeStamps_.value(DetectionInfo::kTimeSkewAffine, 0)); ui_->label_timeExtraction->setNum(info.timeStamps_.value(DetectionInfo::kTimeDescriptorExtraction, 0)); ui_->label_timeSubPix->setNum(info.timeStamps_.value(DetectionInfo::kTimeSubPixelRefining, 0)); - ui_->imageView_source->setData(info.sceneKeypoints_, cvtCvMat2QImage(sceneImage_)); + ui_->imageView_source->updateImage(cvtCvMat2QImage(sceneImage_)); + ui_->imageView_source->updateData(info.sceneKeypoints_, info.sceneWords_); } @@ -1454,6 +1495,7 @@ void MainWindow::notifyParametersChanged(const QStringList & paramChanged) else if( (iter->contains("NearestNeighbor") && Settings::getGeneral_invertedSearch()) || iter->compare(Settings::kGeneral_invertedSearch()) == 0 || (iter->compare(Settings::kGeneral_vocabularyIncremental()) == 0 && Settings::getGeneral_invertedSearch()) || + (iter->compare(Settings::kGeneral_vocabularyFixed()) == 0 && Settings::getGeneral_invertedSearch()) || (iter->compare(Settings::kGeneral_threads()) == 0 && !Settings::getGeneral_invertedSearch()) ) { nearestNeighborParamsChanged = true; @@ -1478,7 +1520,7 @@ void MainWindow::notifyParametersChanged(const QStringList & paramChanged) { if(detectorDescriptorParamsChanged) { - this->updateObjects(objWidgets_.keys()); + this->updateObjects(); } else if(nearestNeighborParamsChanged) { diff --git a/src/ObjWidget.cpp b/src/ObjWidget.cpp index 351421cf..b852a446 100644 --- a/src/ObjWidget.cpp +++ b/src/ObjWidget.cpp @@ -65,7 +65,7 @@ ObjWidget::ObjWidget(QWidget * parent) : { setupUi(); } -ObjWidget::ObjWidget(int id, const std::vector & keypoints, const QImage & image, QWidget * parent) : +ObjWidget::ObjWidget(int id, const std::vector & keypoints, const QMultiMap & words, const QImage & image, QWidget * parent) : QWidget(parent), id_(id), graphicsView_(0), @@ -74,7 +74,8 @@ ObjWidget::ObjWidget(int id, const std::vector & keypoints, const color_(QColor((Qt::GlobalColor)((id % 11 + 7)==Qt::yellow?Qt::gray:(id % 11 + 7)))) { setupUi(); - this->setData(keypoints, image, image.rect()); + this->updateImage(image); + this->updateData(keypoints, words); } ObjWidget::~ObjWidget() { @@ -264,29 +265,42 @@ void ObjWidget::setTextLabel(const QString & text) label_->setText(text); } -void ObjWidget::setData(const std::vector & keypoints, const QImage & image, const QRect & rect) +void ObjWidget::updateImage(const QImage & image) +{ + pixmap_ = QPixmap::fromImage(image); + rect_ = pixmap_.rect(); + label_->setVisible(image.isNull()); +} +void ObjWidget::updateData(const std::vector & keypoints, const QMultiMap & words) { keypoints_ = keypoints; kptColors_ = QVector((int)keypoints.size(), defaultColor()); keypointItems_.clear(); rectItems_.clear(); + this->updateWords(words); graphicsView_->scene()->clear(); graphicsViewInitialized_ = false; mouseCurrentPos_ = mousePressedPos_; // this will reset roi selection - pixmap_ = QPixmap::fromImage(image); - rect_ = rect; - if(rect_.isNull()) - { - rect_ = pixmap_.rect(); - } //this->setMinimumSize(image_.size()); if(graphicsViewMode_->isChecked()) { this->setupGraphicsView(); } - label_->setVisible(image.isNull()); +} + +void ObjWidget::updateWords(const QMultiMap & words) +{ + words_.clear(); + for(QMultiMap::const_iterator iter=words.begin(); iter!=words.end(); ++iter) + { + words_.insert(iter.value(), iter.key()); + } + for(int i=0; isetWordID(words_.value(i,-1)); + } } void ObjWidget::resetKptsColor() @@ -303,6 +317,15 @@ void ObjWidget::resetKptsColor() rectItems_.clear(); } +void ObjWidget::resetKptsWordID() +{ + words_.clear(); + for(int i=0; isetWordID(-1); + } +} + void ObjWidget::setKptColor(int index, const QColor & color) { if(index < kptColors_.size()) @@ -325,6 +348,15 @@ void ObjWidget::setKptColor(int index, const QColor & color) } } +void ObjWidget::setKptWordID(int index, int wordID) +{ + words_.insert(index, wordID); + if(index < keypointItems_.size()) + { + keypointItems_.at(index)->setWordID(wordID); + } +} + void ObjWidget::addRect(QGraphicsRectItem * rect) { if(graphicsViewInitialized_) @@ -701,14 +733,8 @@ void ObjWidget::drawKeypoints(QPainter * painter) QColor color(kptColors_.at(i).red(), kptColors_.at(i).green(), kptColors_.at(i).blue(), alpha_); if(graphicsViewMode_->isChecked()) { - QString info = QString( "ID = %1\n" - "Response = %2\n" - "Angle = %3\n" - "X = %4\n" - "Y = %5\n" - "Size = %6").arg(i+1).arg(r.response).arg(r.angle).arg(r.pt.x).arg(r.pt.y).arg(r.size); // YELLOW = NEW and multiple times - item = new KeypointItem(i+1, r.pt.x-radius, r.pt.y-radius, radius*2, info, color); + item = new KeypointItem(i, r.pt.x-radius, r.pt.y-radius, radius*2, r, words_.value(i, -1), color); item->setVisible(this->isFeaturesShown()); item->setZValue(2); graphicsView_->scene()->addItem(item); @@ -743,7 +769,7 @@ std::vector ObjWidget::selectedKeypoints() const { if(qgraphicsitem_cast(items.at(i))) { - selected.push_back(keypoints_.at(((KeypointItem*)items.at(i))->id()-1)); // ids start at 1 + selected.push_back(keypoints_.at(((KeypointItem*)items.at(i))->id())); } } } diff --git a/src/Vocabulary.h b/src/Vocabulary.h index 8e9360e9..844adecf 100644 --- a/src/Vocabulary.h +++ b/src/Vocabulary.h @@ -44,6 +44,8 @@ public: void update(); void search(const cv::Mat & descriptors, cv::Mat & results, cv::Mat & dists, int k); int size() const {return indexedDescriptors_.rows + notIndexedDescriptors_.rows;} + int dim() const {return !indexedDescriptors_.empty()?indexedDescriptors_.cols:notIndexedDescriptors_.cols;} + int type() const {return !indexedDescriptors_.empty()?indexedDescriptors_.type():notIndexedDescriptors_.type();} const QMultiMap & wordToObjects() const {return wordToObjects_;} const cv::Mat & indexedDescriptors() const {return indexedDescriptors_;}