/* * Copyright (C) 2011, Mathieu Labbe - IntRoLab - Universite de Sherbrooke */ #include "find_object/MainWindow.h" #include "find_object/Camera.h" #include "find_object/Settings.h" #include "find_object/TcpServer.h" #include "find_object/FindObject.h" #include "find_object/utilite/ULogger.h" #include "find_object/ObjWidget.h" #include "find_object/QtOpenCV.h" #include "AddObjectDialog.h" #include "ui_mainWindow.h" #include "KeypointItem.h" #include "RectItem.h" #include "ParametersToolBox.h" #include "AboutDialog.h" #include "rtabmap/PdfPlot.h" #include "Vocabulary.h" #include "ObjSignature.h" #include #include #include "opencv2/calib3d/calib3d.hpp" #include "opencv2/imgproc/imgproc.hpp" #include "opencv2/gpu/gpu.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "utilite/UDirectory.h" // Camera ownership transferred MainWindow::MainWindow(FindObject * findObject, Camera * camera, QWidget * parent) : QMainWindow(parent), camera_(camera), findObject_(findObject), likelihoodCurve_(0), inliersCurve_(0), lowestRefreshRate_(99), objectsModified_(false), tcpServer_(0) { Q_ASSERT(findObject_ != 0); ui_ = new Ui_mainWindow(); ui_->setupUi(this); aboutDialog_ = new AboutDialog(this); this->setStatusBar(new QStatusBar()); likelihoodCurve_ = new rtabmap::PdfPlotCurve("Likelihood", &imagesMap_, this); inliersCurve_ = new rtabmap::PdfPlotCurve("Inliers", &imagesMap_, this); likelihoodCurve_->setPen(QPen(Qt::blue)); inliersCurve_->setPen(QPen(Qt::red)); ui_->likelihoodPlot->addCurve(likelihoodCurve_, false); ui_->likelihoodPlot->addCurve(inliersCurve_, false); ui_->likelihoodPlot->setGraphicsView(true); ui_->dockWidget_statistics->setVisible(false); ui_->dockWidget_parameters->setVisible(false); ui_->dockWidget_plot->setVisible(false); ui_->widget_controls->setVisible(false); QByteArray geometry; QByteArray state; Settings::loadWindowSettings(geometry, state); this->restoreGeometry(geometry); this->restoreState(state); lastObjectsUpdateParameters_ = Settings::getParameters(); ui_->toolBox->setupUi(); if(!camera_) { camera_ = new Camera(this); } else { camera_->setParent(this); ui_->toolBox->getParameterWidget(Settings::kCamera_1deviceId())->setEnabled(false); ui_->toolBox->getParameterWidget(Settings::kCamera_2imageWidth())->setEnabled(false); ui_->toolBox->getParameterWidget(Settings::kCamera_3imageHeight())->setEnabled(false); ui_->toolBox->getParameterWidget(Settings::kCamera_5mediaPath())->setEnabled(false); ui_->toolBox->getParameterWidget(Settings::kCamera_6useTcpCamera())->setEnabled(false); ui_->toolBox->getParameterWidget(Settings::kCamera_8port())->setEnabled(false); ui_->toolBox->getParameterWidget(Settings::kCamera_9queueSize())->setEnabled(false); ui_->actionCamera_from_video_file->setVisible(false); ui_->actionCamera_from_TCP_IP->setVisible(false); ui_->actionCamera_from_directory_of_images->setVisible(false); ui_->actionLoad_scene_from_file->setVisible(false); } if(cv::gpu::getCudaEnabledDeviceCount() == 0) { ui_->toolBox->updateParameter(Settings::kFeature2D_SURF_gpu()); ui_->toolBox->updateParameter(Settings::kFeature2D_Fast_gpu()); ui_->toolBox->updateParameter(Settings::kFeature2D_ORB_gpu()); ui_->toolBox->getParameterWidget(Settings::kFeature2D_SURF_gpu())->setEnabled(false); ui_->toolBox->getParameterWidget(Settings::kFeature2D_SURF_keypointsRatio())->setEnabled(false); ui_->toolBox->getParameterWidget(Settings::kFeature2D_Fast_gpu())->setEnabled(false); ui_->toolBox->getParameterWidget(Settings::kFeature2D_Fast_keypointsRatio())->setEnabled(false); ui_->toolBox->getParameterWidget(Settings::kFeature2D_ORB_gpu())->setEnabled(false); } connect((QDoubleSpinBox*)ui_->toolBox->getParameterWidget(Settings::kCamera_4imageRate()), SIGNAL(editingFinished()), camera_, SLOT(updateImageRate())); ui_->menuView->addAction(ui_->dockWidget_statistics->toggleViewAction()); ui_->menuView->addAction(ui_->dockWidget_parameters->toggleViewAction()); ui_->menuView->addAction(ui_->dockWidget_objects->toggleViewAction()); ui_->menuView->addAction(ui_->dockWidget_plot->toggleViewAction()); connect(ui_->toolBox, SIGNAL(parametersChanged(const QStringList &)), this, SLOT(notifyParametersChanged(const QStringList &))); ui_->imageView_source->setGraphicsViewMode(false); ui_->imageView_source->setTextLabel(tr("Press \"space\" to start the camera...")); ui_->imageView_source->setMirrorView(Settings::getGeneral_mirrorView()); connect((QCheckBox*)ui_->toolBox->getParameterWidget(Settings::kGeneral_mirrorView()), SIGNAL(stateChanged(int)), this, SLOT(updateMirrorView())); ui_->widget_controls->setVisible(Settings::getGeneral_controlsShown()); connect((QCheckBox*)ui_->toolBox->getParameterWidget(Settings::kGeneral_controlsShown()), SIGNAL(stateChanged(int)), this, SLOT(showHideControls())); //buttons connect(ui_->pushButton_restoreDefaults, SIGNAL(clicked()), ui_->toolBox, SLOT(resetCurrentPage())); connect(ui_->pushButton_updateObjects, SIGNAL(clicked()), this, SLOT(updateObjects())); connect(ui_->horizontalSlider_objectsSize, SIGNAL(valueChanged(int)), this, SLOT(updateObjectsSize())); ui_->actionStop_camera->setEnabled(false); ui_->actionPause_camera->setEnabled(false); ui_->actionSave_objects->setEnabled(false); // Actions connect(ui_->actionAdd_object_from_scene, SIGNAL(triggered()), this, SLOT(addObjectFromScene())); connect(ui_->actionAdd_objects_from_files, SIGNAL(triggered()), this, SLOT(addObjectsFromFiles())); connect(ui_->actionLoad_scene_from_file, SIGNAL(triggered()), this, SLOT(loadSceneFromFile())); connect(ui_->actionStart_camera, SIGNAL(triggered()), this, SLOT(startProcessing())); connect(ui_->actionStop_camera, SIGNAL(triggered()), this, SLOT(stopProcessing())); connect(ui_->actionPause_camera, SIGNAL(triggered()), this, SLOT(pauseProcessing())); connect(ui_->actionExit, SIGNAL(triggered()), this, SLOT(close())); connect(ui_->actionSave_objects, SIGNAL(triggered()), this, SLOT(saveObjects())); connect(ui_->actionLoad_objects, SIGNAL(triggered()), this, SLOT(loadObjects())); connect(ui_->actionCamera_from_video_file, SIGNAL(triggered()), this, SLOT(setupCameraFromVideoFile())); connect(ui_->actionCamera_from_directory_of_images, SIGNAL(triggered()), this, SLOT(setupCameraFromImagesDirectory())); connect(ui_->actionCamera_from_TCP_IP, SIGNAL(triggered()), this, SLOT(setupCameraFromTcpIp())); connect(ui_->actionAbout, SIGNAL(triggered()), aboutDialog_ , SLOT(exec())); connect(ui_->actionRestore_all_default_settings, SIGNAL(triggered()), ui_->toolBox, SLOT(resetAllPages())); connect(ui_->actionRemove_all_objects, SIGNAL(triggered()), this, SLOT(removeAllObjects())); connect(ui_->actionSave_settings, SIGNAL(triggered()), this, SLOT(saveSettings())); connect(ui_->actionLoad_settings, SIGNAL(triggered()), this, SLOT(loadSettings())); connect(ui_->pushButton_play, SIGNAL(clicked()), this, SLOT(startProcessing())); connect(ui_->pushButton_stop, SIGNAL(clicked()), this, SLOT(stopProcessing())); connect(ui_->pushButton_pause, SIGNAL(clicked()), this, SLOT(pauseProcessing())); connect(ui_->horizontalSlider_frames, SIGNAL(valueChanged(int)), this, SLOT(moveCameraFrame(int))); connect(ui_->horizontalSlider_frames, SIGNAL(valueChanged(int)), ui_->label_frame, SLOT(setNum(int))); ui_->pushButton_play->setVisible(true); ui_->pushButton_pause->setVisible(false); ui_->pushButton_stop->setEnabled(false); ui_->horizontalSlider_frames->setEnabled(false); ui_->label_frame->setVisible(false); ui_->objects_area->addAction(ui_->actionAdd_object_from_scene); ui_->objects_area->addAction(ui_->actionAdd_objects_from_files); ui_->objects_area->setContextMenuPolicy(Qt::ActionsContextMenu); ui_->actionStart_camera->setShortcut(Qt::Key_Space); ui_->actionPause_camera->setShortcut(Qt::Key_Space); ui_->actionCamera_from_video_file->setChecked(!Settings::getCamera_5mediaPath().isEmpty() && !UDirectory::exists(Settings::getCamera_5mediaPath().toStdString())); ui_->actionCamera_from_directory_of_images->setChecked(!Settings::getCamera_5mediaPath().isEmpty() && UDirectory::exists(Settings::getCamera_5mediaPath().toStdString())); ui_->actionCamera_from_TCP_IP->setChecked(Settings::getCamera_6useTcpCamera()); ui_->label_ipAddress->setTextInteractionFlags(Qt::TextSelectableByMouse); ui_->label_port->setTextInteractionFlags(Qt::TextSelectableByMouse); setupTCPServer(); if(findObject_->objects().size()) { // show objects already loaded in FindObject for(QMap::const_iterator iter = findObject_->objects().constBegin(); iter!=findObject_->objects().constEnd(); ++iter) { ObjWidget * obj = new ObjWidget(iter.key(), iter.value()->keypoints(), cvtCvMat2QImage(iter.value()->image())); objWidgets_.insert(obj->id(), obj); this->showObject(obj); } } if(Settings::getGeneral_autoStartCamera()) { // Set 1 msec to see state on the status bar. QTimer::singleShot(1, this, SLOT(startProcessing())); } } MainWindow::~MainWindow() { disconnect(camera_, SIGNAL(imageReceived(const cv::Mat &)), this, SLOT(update(const cv::Mat &))); camera_->stop(); qDeleteAll(objWidgets_); objWidgets_.clear(); delete ui_; delete findObject_; } void MainWindow::closeEvent(QCloseEvent * event) { bool quit = true; this->stopProcessing(); if(objectsModified_ && this->isVisible() && objWidgets_.size()) { int ret = QMessageBox::question(this, tr("Save new objects"), tr("Do you want to save added objects?"), QMessageBox::Yes | QMessageBox::No | QMessageBox::Cancel); switch(ret) { case QMessageBox::Yes: quit = this->saveObjects(); break; case QMessageBox::Cancel: quit = false; break; case QMessageBox::No: default: break; } } if(quit) { Settings::saveWindowSettings(this->saveGeometry(), this->saveState()); event->accept(); } else { event->ignore(); } } void MainWindow::setupTCPServer() { if(tcpServer_) { tcpServer_->close(); delete tcpServer_; } tcpServer_ = new TcpServer(Settings::getGeneral_port(), this); connect(this, SIGNAL(objectsFound(QMultiMap >)), tcpServer_, SLOT(publishObjects(QMultiMap >))); ui_->label_ipAddress->setText(tcpServer_->getHostAddress().toString()); ui_->label_port->setNum(tcpServer_->getPort()); UINFO("Detection sent on port: %d (IP=%s)", tcpServer_->getPort(), tcpServer_->getHostAddress().toString().toStdString().c_str()); } void MainWindow::setSourceImageText(const QString & text) { ui_->imageView_source->setTextLabel(text); } void MainWindow::loadSettings() { QString path = QFileDialog::getOpenFileName(this, tr("Load settings..."), Settings::workingDirectory(), "*.ini"); if(!path.isEmpty()) { if(QFileInfo(path).suffix().compare("ini") != 0) { path.append(".ini"); } loadSettings(path); } } void MainWindow::saveSettings() { QString path = QFileDialog::getSaveFileName(this, tr("Save settings..."), Settings::workingDirectory(), "*.ini"); if(!path.isEmpty()) { if(QFileInfo(path).suffix().compare("ini") != 0) { path.append(".ini"); } saveSettings(path); } } bool MainWindow::loadSettings(const QString & path) { if(!path.isEmpty() && QFileInfo(path).suffix().compare("ini") == 0) { QByteArray geometry; QByteArray state; Settings::loadSettings(path); Settings::loadWindowSettings(geometry, state, path); this->restoreGeometry(geometry); this->restoreState(state); //update parameters tool box const ParametersMap & parameters = Settings::getParameters(); for(ParametersMap::const_iterator iter = parameters.begin(); iter!= parameters.constEnd(); ++iter) { ui_->toolBox->updateParameter(iter.key()); } return true; } UINFO("Path \"%s\" not valid (should be *.ini)", path.toStdString().c_str()); return false; } bool MainWindow::saveSettings(const QString & path) { if(!path.isEmpty() && QFileInfo(path).suffix().compare("ini") == 0) { Settings::saveSettings(path); Settings::saveWindowSettings(this->saveGeometry(), this->saveState(), path); return true; } UINFO("Path \"%s\" not valid (should be *.ini)", path.toStdString().c_str()); return false; } int MainWindow::loadObjects(const QString & dirPath) { int loadedObjects = 0; QString formats = Settings::getGeneral_imageFormats().remove('*').remove('.'); UDirectory dir(dirPath.toStdString(), formats.toStdString()); if(dir.isValid()) { const std::list & names = dir.getFileNames(); // sorted in natural order for(std::list::const_iterator iter=names.begin(); iter!=names.end(); ++iter) { if(this->addObjectFromFile((dirPath.toStdString()+dir.separator()+*iter).c_str())) { ++loadedObjects; } } if(loadedObjects) { this->updateObjects(); } } return loadedObjects; } int MainWindow::saveObjects(const QString & dirPath) { int count = 0; QDir dir(dirPath); if(dir.exists()) { for(QMap::iterator iter=objWidgets_.begin(); iter!=objWidgets_.end(); ++iter) { if(iter.value()->pixmap().save(QString("%1/%2.png").arg(dirPath).arg(iter.key()))) { ++count; } else { UERROR("Failed to save object %d", iter.key()); } } objectsModified_ = false; } return count; } void MainWindow::loadObjects() { QString dirPath = QFileDialog::getExistingDirectory(this, tr("Loading objects... Select a directory."), Settings::workingDirectory()); if(!dirPath.isEmpty()) { int count = loadObjects(dirPath); if(count) { QMessageBox::information(this, tr("Loading..."), tr("%1 objects loaded from \"%2\".").arg(count).arg(dirPath)); } else { QMessageBox::information(this, tr("Loading..."), tr("No objects loaded from \"%1\"!").arg(dirPath)); } } } bool MainWindow::saveObjects() { QString dirPath = QFileDialog::getExistingDirectory(this, tr("Saving objects... Select a directory."), Settings::workingDirectory()); if(!dirPath.isEmpty()) { int count = saveObjects(dirPath); if(count) { QMessageBox::information(this, tr("Saving..."), tr("%1 objects saved to \"%2\".").arg(count).arg(dirPath)); } else { QMessageBox::warning(this, tr("Saving..."), tr("No objects saved to %1!").arg(dirPath)); } return count > 0; } return false; } void MainWindow::removeObject(ObjWidget * object) { if(object) { objWidgets_.remove(object->id()); if(objWidgets_.size() == 0) { ui_->actionSave_objects->setEnabled(false); } findObject_->removeObject(object->id()); object->deleteLater(); if(Settings::getGeneral_autoUpdateObjects()) { this->updateVocabulary(); } if(!camera_->isRunning() && !sceneImage_.empty()) { this->update(); } } } void MainWindow::removeAllObjects() { qDeleteAll(objWidgets_); objWidgets_.clear(); ui_->actionSave_objects->setEnabled(false); findObject_->removeAllObjects(); if(!camera_->isRunning() && !sceneImage_.empty()) { this->update(); } } void MainWindow::updateObjectsSize() { for(QMap::iterator iter=objWidgets_.begin(); iter!=objWidgets_.end(); ++iter) { updateObjectSize(iter.value()); } } void MainWindow::updateObjectSize(ObjWidget * obj) { if(obj) { int value = ui_->horizontalSlider_objectsSize->value(); if((obj->pixmap().width()*value)/100 > 4 && (obj->pixmap().height()*value)/100 > 4) { obj->setVisible(true); obj->setMinimumSize((obj->pixmap().width()*value)/100, (obj->pixmap().height())*value/100); } else { obj->setVisible(false); } } } void MainWindow::updateMirrorView() { bool mirrorView = Settings::getGeneral_mirrorView(); ui_->imageView_source->setMirrorView(mirrorView); for(QMap::iterator iter=objWidgets_.begin(); iter!=objWidgets_.end(); ++iter) { iter.value()->setMirrorView(mirrorView); } } void MainWindow::showHideControls() { ui_->widget_controls->setVisible(Settings::getGeneral_controlsShown()); } void MainWindow::addObjectFromScene() { disconnect(camera_, SIGNAL(imageReceived(const cv::Mat &)), this, SLOT(update(const cv::Mat &))); AddObjectDialog * dialog; bool resumeCamera = camera_->isRunning(); if(camera_->isRunning() || sceneImage_.empty()) { dialog = new AddObjectDialog(camera_, cv::Mat(), ui_->imageView_source->isMirrorView(), this); } else { dialog = new AddObjectDialog(0, sceneImage_, ui_->imageView_source->isMirrorView(), this); } if(dialog->exec() == QDialog::Accepted) { ObjWidget * obj = 0; ObjSignature * signature = 0; dialog->retrieveObject(&obj, &signature); Q_ASSERT(obj!=0 && signature!=0); findObject_->addObject(signature); obj->setId(signature->id()); objWidgets_.insert(obj->id(), obj); ui_->actionSave_objects->setEnabled(true); showObject(obj); QLabel * detectorDescriptorType = qFindChild(this, QString("%1type").arg(obj->id())); detectorDescriptorType->setText(QString("%1/%2").arg(signature->detectorType()).arg(signature->descriptorType())); updateVocabulary(); objectsModified_ = true; } if(resumeCamera || sceneImage_.empty()) { this->startProcessing(); } else { connect(camera_, SIGNAL(imageReceived(const cv::Mat &)), this, SLOT(update(const cv::Mat &)), Qt::UniqueConnection); this->update(); } delete dialog; } void MainWindow::addObjectsFromFiles() { QStringList fileNames = QFileDialog::getOpenFileNames(this, tr("Add objects..."), Settings::workingDirectory(), tr("Image Files (%1)").arg(Settings::getGeneral_imageFormats())); if(fileNames.size()) { for(int i=0; iaddObjectFromFile(fileNames.at(i)); } objectsModified_ = true; updateObjects(); } } bool MainWindow::addObjectFromFile(const QString & filePath) { const ObjSignature * s = findObject_->addObject(filePath); if(s) { ObjWidget * obj = new ObjWidget(s->id(), std::vector(), cvtCvMat2QImage(s->image())); objWidgets_.insert(obj->id(), obj); ui_->actionSave_objects->setEnabled(true); this->showObject(obj); return true; } else { QMessageBox::critical(this, tr("Error adding object"), tr("Failed to add object from \"%1\"").arg(filePath)); return false; } } void MainWindow::loadSceneFromFile() { QString fileName = QFileDialog::getOpenFileName(this, tr("Load scene..."), Settings::workingDirectory(), tr("Image Files (%1)").arg(Settings::getGeneral_imageFormats())); if(!fileName.isEmpty()) { cv::Mat img = cv::imread(fileName.toStdString().c_str()); if(!img.empty()) { this->update(img); ui_->label_timeRefreshRate->setVisible(false); } } } void MainWindow::setupCameraFromVideoFile() { if(!ui_->actionCamera_from_video_file->isChecked()) { Settings::setCamera_5mediaPath(""); ui_->toolBox->updateParameter(Settings::kCamera_5mediaPath()); } else { QString fileName = QFileDialog::getOpenFileName(this, tr("Setup camera from video file..."), Settings::workingDirectory(), tr("Video Files (%1)").arg(Settings::getGeneral_videoFormats())); if(!fileName.isEmpty()) { Settings::setCamera_5mediaPath(fileName); ui_->toolBox->updateParameter(Settings::kCamera_5mediaPath()); if(camera_->isRunning()) { this->stopProcessing(); this->startProcessing(); } Settings::setGeneral_controlsShown(true); ui_->toolBox->updateParameter(Settings::kGeneral_controlsShown()); } } ui_->actionCamera_from_video_file->setChecked(!Settings::getCamera_5mediaPath().isEmpty()); ui_->actionCamera_from_directory_of_images->setChecked(false); ui_->actionCamera_from_TCP_IP->setChecked(false); } void MainWindow::setupCameraFromImagesDirectory() { if(!ui_->actionCamera_from_directory_of_images->isChecked()) { Settings::setCamera_5mediaPath(""); ui_->toolBox->updateParameter(Settings::kCamera_5mediaPath()); } else { QString directory = QFileDialog::getExistingDirectory(this, tr("Setup camera from directory of images..."), Settings::workingDirectory()); if(!directory.isEmpty()) { Settings::setCamera_5mediaPath(directory); ui_->toolBox->updateParameter(Settings::kCamera_5mediaPath()); if(camera_->isRunning()) { this->stopProcessing(); this->startProcessing(); } Settings::setGeneral_controlsShown(true); ui_->toolBox->updateParameter(Settings::kGeneral_controlsShown()); } } ui_->actionCamera_from_directory_of_images->setChecked(!Settings::getCamera_5mediaPath().isEmpty()); ui_->actionCamera_from_video_file->setChecked(false); ui_->actionCamera_from_TCP_IP->setChecked(false); } void MainWindow::setupCameraFromTcpIp() { if(!ui_->actionCamera_from_TCP_IP->isChecked()) { Settings::setCamera_6useTcpCamera(false); ui_->toolBox->updateParameter(Settings::kCamera_6useTcpCamera()); } else { bool ok; int port = QInputDialog::getInteger(this, tr("Server port..."), "Port: ", Settings::getCamera_8port(), 1, USHRT_MAX, 1, &ok); if(ok) { int queue = QInputDialog::getInteger(this, tr("Queue size..."), "Images buffer size (0 means infinite): ", Settings::getCamera_9queueSize(), 0, 2147483647, 1, &ok); if(ok) { Settings::setCamera_6useTcpCamera(true); ui_->toolBox->updateParameter(Settings::kCamera_6useTcpCamera()); Settings::setCamera_8port(port); ui_->toolBox->updateParameter(Settings::kCamera_8port()); Settings::setCamera_9queueSize(queue); ui_->toolBox->updateParameter(Settings::kCamera_9queueSize()); if(camera_->isRunning()) { this->stopProcessing(); } this->startProcessing(); } } } ui_->actionCamera_from_directory_of_images->setChecked(false); ui_->actionCamera_from_video_file->setChecked(false); ui_->actionCamera_from_TCP_IP->setChecked(Settings::getCamera_6useTcpCamera()); } void MainWindow::showObject(ObjWidget * obj) { if(obj) { obj->setGraphicsViewMode(false); obj->setMirrorView(ui_->imageView_source->isMirrorView()); QList objs = ui_->objects_area->findChildren(); QVBoxLayout * vLayout = new QVBoxLayout(); ui_->toolBox->updateParameter(Settings::kGeneral_nextObjID()); QLabel * title = new QLabel(QString("%1 (%2)").arg(obj->id()).arg(obj->keypoints().size()), this); QLabel * detectorDescriptorType = new QLabel(QString("%1/%2").arg("detector").arg("descriptor"), this); QLabel * detectedLabel = new QLabel(this); title->setObjectName(QString("%1title").arg(obj->id())); detectorDescriptorType->setObjectName(QString("%1type").arg(obj->id())); detectedLabel->setObjectName(QString("%1detection").arg(obj->id())); QHBoxLayout * hLayout = new QHBoxLayout(); hLayout->addWidget(title); hLayout->addStretch(1); hLayout->addWidget(detectorDescriptorType); hLayout->addStretch(1); hLayout->addWidget(detectedLabel); vLayout->addLayout(hLayout); vLayout->addWidget(obj); obj->setDeletable(true); connect(obj, SIGNAL(removalTriggered(ObjWidget*)), this, SLOT(removeObject(ObjWidget*))); connect(obj, SIGNAL(destroyed(QObject *)), title, SLOT(deleteLater())); connect(obj, SIGNAL(destroyed(QObject *)), detectedLabel, SLOT(deleteLater())); connect(obj, SIGNAL(destroyed(QObject *)), detectorDescriptorType, SLOT(deleteLater())); connect(obj, SIGNAL(destroyed(QObject *)), vLayout, SLOT(deleteLater())); ui_->verticalLayout_objects->insertLayout(ui_->verticalLayout_objects->count()-1, vLayout); QByteArray ba; QBuffer buffer(&ba); buffer.open(QIODevice::WriteOnly); obj->pixmap().scaledToWidth(128).save(&buffer, "JPEG"); // writes image into JPEG format imagesMap_.insert(obj->id(), ba); updateObjectSize(obj); } } void MainWindow::updateObjects() { this->statusBar()->showMessage(tr("Updating %1 objects...").arg(findObject_->objects().size())); QApplication::processEvents(); findObject_->updateObjects(); updateVocabulary(); QList signatures = findObject_->objects().values(); for(int i=0; iid())->setData(signatures[i]->keypoints(), cvtCvMat2QImage(signatures[i]->image())); //update object labels QLabel * title = qFindChild(this, QString("%1title").arg(signatures[i]->id())); title->setText(QString("%1 (%2)").arg(signatures[i]->id()).arg(QString::number(signatures[i]->keypoints().size()))); QLabel * detectorDescriptorType = qFindChild(this, QString("%1type").arg(signatures[i]->id())); detectorDescriptorType->setText(QString("%1/%2").arg(signatures[i]->detectorType()).arg(signatures[i]->descriptorType())); } if(!camera_->isRunning() && !sceneImage_.empty()) { this->update(); } this->statusBar()->clearMessage(); } void MainWindow::updateVocabulary() { this->statusBar()->showMessage(tr("Updating vocabulary...")); QApplication::processEvents(); QTime time; findObject_->updateVocabulary(); if(findObject_->vocabulary()->size()) { ui_->label_timeIndexing->setNum(time.elapsed()); ui_->label_vocabularySize->setNum(findObject_->vocabulary()->size()); } lastObjectsUpdateParameters_ = Settings::getParameters(); this->statusBar()->clearMessage(); } void MainWindow::startProcessing() { UINFO("Starting camera..."); bool updateStatusMessage = this->statusBar()->currentMessage().isEmpty(); if(updateStatusMessage) { this->statusBar()->showMessage(tr("Starting camera...")); } if(camera_->start()) { connect(camera_, SIGNAL(imageReceived(const cv::Mat &)), this, SLOT(update(const cv::Mat &)), Qt::UniqueConnection); ui_->actionStop_camera->setEnabled(true); ui_->actionPause_camera->setEnabled(true); ui_->actionStart_camera->setEnabled(false); ui_->actionLoad_scene_from_file->setEnabled(false); ui_->actionCamera_from_directory_of_images->setEnabled(false); ui_->actionCamera_from_video_file->setEnabled(false); ui_->actionCamera_from_TCP_IP->setEnabled(false); ui_->label_timeRefreshRate->setVisible(true); //update control bar ui_->pushButton_play->setVisible(false); ui_->pushButton_pause->setVisible(true); ui_->pushButton_stop->setEnabled(true); int totalFrames = camera_->getTotalFrames(); if(totalFrames>0) { ui_->label_frame->setVisible(true); ui_->horizontalSlider_frames->setEnabled(true); ui_->horizontalSlider_frames->setMaximum(totalFrames-1); } //update camera port if TCP is used ui_->label_port_image->setText("-"); if(Settings::getCamera_6useTcpCamera() && camera_->getPort()) { ui_->label_port_image->setNum(camera_->getPort()); } if(updateStatusMessage) { this->statusBar()->showMessage(tr("Camera started."), 2000); } } else { if(updateStatusMessage) { this->statusBar()->clearMessage(); } if(Settings::getCamera_6useTcpCamera()) { QMessageBox::critical(this, tr("Camera error"), tr("Camera initialization failed! (with port %1)").arg(Settings::getCamera_8port())); } else { QMessageBox::critical(this, tr("Camera error"), tr("Camera initialization failed! (with device %1)").arg(Settings::getCamera_1deviceId())); } } } void MainWindow::stopProcessing() { if(camera_) { disconnect(camera_, SIGNAL(imageReceived(const cv::Mat &)), this, SLOT(update(const cv::Mat &))); camera_->stop(); } ui_->actionStop_camera->setEnabled(false); ui_->actionPause_camera->setEnabled(false); ui_->actionStart_camera->setEnabled(true); ui_->actionLoad_scene_from_file->setEnabled(true); ui_->actionCamera_from_directory_of_images->setEnabled(true); ui_->actionCamera_from_video_file->setEnabled(true); ui_->actionCamera_from_TCP_IP->setEnabled(true); ui_->pushButton_play->setVisible(true); ui_->pushButton_pause->setVisible(false); ui_->pushButton_stop->setEnabled(false); ui_->horizontalSlider_frames->setEnabled(false); ui_->horizontalSlider_frames->setValue(0); ui_->label_frame->setVisible(false); ui_->label_port_image->setText("-"); } void MainWindow::pauseProcessing() { ui_->actionStop_camera->setEnabled(true); ui_->actionPause_camera->setEnabled(true); ui_->actionStart_camera->setEnabled(false); if(camera_->isRunning()) { ui_->pushButton_play->setVisible(true); ui_->pushButton_pause->setVisible(false); camera_->pause(); } else { ui_->pushButton_play->setVisible(false); ui_->pushButton_pause->setVisible(true); camera_->start(); } } void MainWindow::moveCameraFrame(int frame) { if(ui_->horizontalSlider_frames->isEnabled()) { camera_->moveToFrame(frame); if(!camera_->isRunning()) { camera_->takeImage(); } } } void MainWindow::rectHovered(int objId) { if(objId>=0 && Settings::getGeneral_autoScroll()) { QLabel * label = ui_->dockWidget_objects->findChild(QString("%1title").arg(objId)); if(label) { ui_->objects_area->verticalScrollBar()->setValue(label->pos().y()); } } } void MainWindow::update(const cv::Mat & image) { if(image.empty()) { UWARN("Camera cannot get more images (maybe the end of stream is reached)..."); return; } // reset objects color for(QMap::iterator iter=objWidgets_.begin(); iter!=objWidgets_.end(); ++iter) { iter.value()->resetKptsColor(); } if(!image.empty()) { sceneImage_ = image.clone(); } QMultiMap > objectsDetected; if(findObject_->detect(sceneImage_, objectsDetected)) { ui_->label_timeDetection->setNum(findObject_->timeStamps().value(FindObject::kTimeKeypointDetection, 0)); ui_->label_timeExtraction->setNum(findObject_->timeStamps().value(FindObject::kTimeDescriptorExtraction, 0)); ui_->imageView_source->setData(findObject_->sceneKeypoints(), cvtCvMat2QImage(sceneImage_)); ui_->label_timeIndexing->setNum(findObject_->timeStamps().value(FindObject::kTimeIndexing, 0)); ui_->label_vocabularySize->setNum(findObject_->vocabulary()->size()); ui_->label_timeMatching->setNum(findObject_->timeStamps().value(FindObject::kTimeMatching, 0)); ui_->label_timeHomographies->setNum(findObject_->timeStamps().value(FindObject::kTimeHomography, 0)); // Colorize features matched const QMap > & matches = findObject_->matches(); QMap scores; int maxScoreId = -1; int maxScore = 0; for(QMap >::const_iterator jter=matches.constBegin(); jter!=matches.end();++jter) { scores.insert(jter.key(), jter.value().size()); if(maxScoreId == -1 || maxScore < jter.value().size()) { maxScoreId = jter.key(); maxScore = jter.value().size(); } int id = jter.key(); QLabel * label = ui_->dockWidget_objects->findChild(QString("%1detection").arg(id)); if(!Settings::getHomography_homographyComputed()) { label->setText(QString("%1 matches").arg(jter.value().size())); ObjWidget * obj = objWidgets_.value(id); Q_ASSERT(obj != 0); int nColor = id % 11 + 7; QColor color((Qt::GlobalColor)(nColor==Qt::yellow?Qt::gray:nColor)); for(QMultiMap::const_iterator iter = jter.value().constBegin(); iter!= jter.value().constEnd(); ++iter) { obj->setKptColor(iter.key(), color); ui_->imageView_source->setKptColor(iter.value(), color); } } else if(!objectsDetected.contains(id)) { // Homography could not be computed... QLabel * label = ui_->dockWidget_objects->findChild(QString("%1detection").arg(id)); QMultiMap rejectedInliers = findObject_->rejectedInliers().value(id); QMultiMap rejectedOutliers = findObject_->rejectedOutliers().value(id); if(jter.value().size() < Settings::getHomography_minimumInliers()) { label->setText(QString("Too low matches (%1)").arg(jter.value().size())); } else if(rejectedInliers.size() >= Settings::getHomography_minimumInliers()) { label->setText(QString("Ignored, all inliers (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size())); } else { label->setText(QString("Too low inliers (%1 in %2 out)").arg(rejectedInliers.size()).arg(rejectedOutliers.size())); } } } // Add homography rectangles when homographies are computed QMultiMap >::const_iterator inliersIter = findObject_->inliers().constBegin(); QMultiMap >::const_iterator outliersIter = findObject_->outliers().constBegin(); for(QMultiMap >::iterator iter = objectsDetected.begin(); iter!=objectsDetected.end() && inliersIter!=findObject_->inliers().constEnd(); ++iter, ++inliersIter) { int id = iter.key(); ObjWidget * obj = objWidgets_.value(id); Q_ASSERT(obj != 0); // COLORIZE (should be done in the GUI thread) int nColor = id % 11 + 7; QColor color((Qt::GlobalColor)(nColor==Qt::yellow?Qt::gray:nColor)); QTransform hTransform = iter.value().second; QRect rect = obj->pixmap().rect(); // add rectangle QPen rectPen(color); rectPen.setWidth(Settings::getHomography_rectBorderWidth()); RectItem * rectItemScene = new RectItem(id, rect); connect(rectItemScene, SIGNAL(hovered(int)), this, SLOT(rectHovered(int))); rectItemScene->setPen(rectPen); rectItemScene->setTransform(hTransform); ui_->imageView_source->addRect(rectItemScene); QGraphicsRectItem * rectItemObj = new QGraphicsRectItem(rect); rectItemObj->setPen(rectPen); obj->addRect(rectItemObj); for(QMultiMap::const_iterator iter = inliersIter.value().constBegin(); iter!= inliersIter.value().constEnd(); ++iter) { obj->setKptColor(iter.key(), color); ui_->imageView_source->setKptColor(iter.value(), color); } QLabel * label = ui_->dockWidget_objects->findChild(QString("%1detection").arg(id)); if(objectsDetected.count(id) > 1) { // if a homography is already found, set the objects count label->setText(QString("%1 objects found").arg(objectsDetected.count(id))); } else { label->setText(QString("%1 in %2 out").arg(inliersIter.value().size()).arg(outliersIter.value().size())); } } //update likelihood plot likelihoodCurve_->setData(scores, QMap()); QMap inlierScores; for(QMap::iterator iter=scores.begin(); iter!=scores.end(); ++iter) { QList > values = findObject_->inliers().values(iter.key()); int maxValue = 0; if(values.size()) { maxValue = values[0].size(); for(int i=1; isetData(inlierScores, QMap()); if(ui_->likelihoodPlot->isVisible()) { ui_->likelihoodPlot->update(); } ui_->label_minMatchedDistance->setNum(findObject_->minMatchedDistance()); ui_->label_maxMatchedDistance->setNum(findObject_->maxMatchedDistance()); //Scroll objects slider to the best score if(maxScoreId>=0 && Settings::getGeneral_autoScroll()) { QLabel * label = ui_->dockWidget_objects->findChild(QString("%1title").arg(maxScoreId)); if(label) { ui_->objects_area->verticalScrollBar()->setValue(label->pos().y()); } } // Emit homographies if(objectsDetected.size() > 1) { UINFO("(%s) %d objects detected!", QTime::currentTime().toString("HH:mm:ss.zzz").toStdString().c_str(), (int)objectsDetected.size()); } else if(objectsDetected.size() == 1) { UINFO("(%s) Object %d detected!", QTime::currentTime().toString("HH:mm:ss.zzz").toStdString().c_str(), (int)objectsDetected.begin().key()); } else if(Settings::getGeneral_sendNoObjDetectedEvents()) { UINFO("(%s) No objects detected.", QTime::currentTime().toString("HH:mm:ss.zzz").toStdString().c_str()); } if(objectsDetected.size() > 0 || Settings::getGeneral_sendNoObjDetectedEvents()) { Q_EMIT objectsFound(objectsDetected); } ui_->label_objectsDetected->setNum(objectsDetected.size()); } else { this->statusBar()->showMessage(tr("Cannot search, objects must be updated!")); ui_->imageView_source->setData(findObject_->sceneKeypoints(), cvtCvMat2QImage(sceneImage_)); } //Update object pictures for(QMap::iterator iter=objWidgets_.begin(); iter!=objWidgets_.end(); ++iter) { iter.value()->update(); } ui_->label_nfeatures->setNum((int)findObject_->sceneKeypoints().size()); ui_->imageView_source->update(); ui_->label_detectorDescriptorType->setText(QString("%1/%2").arg(Settings::currentDetectorType()).arg(Settings::currentDescriptorType())); //update slider if(ui_->horizontalSlider_frames->isEnabled()) { ui_->horizontalSlider_frames->blockSignals(true); ui_->horizontalSlider_frames->setValue(camera_->getCurrentFrameIndex()-1); ui_->label_frame->setNum(camera_->getCurrentFrameIndex()-1); ui_->horizontalSlider_frames->blockSignals(false); } ui_->label_timeTotal->setNum(findObject_->timeStamps().value(FindObject::kTimeTotal, 0)); int refreshRate = qRound(1000.0f/float(updateRate_.restart())); if(refreshRate > 0 && refreshRate < lowestRefreshRate_) { lowestRefreshRate_ = refreshRate; } // Refresh the label only after each 1000 ms if(refreshStartTime_.elapsed() > 1000) { if(Settings::getCamera_4imageRate()>0.0) { ui_->label_timeRefreshRate->setText(QString("(%1 Hz - %2 Hz)").arg(QString::number(Settings::getCamera_4imageRate())).arg(QString::number(lowestRefreshRate_))); } else { ui_->label_timeRefreshRate->setText(QString("(%2 Hz)").arg(QString::number(lowestRefreshRate_))); } lowestRefreshRate_ = 99; refreshStartTime_.start(); } } void MainWindow::notifyParametersChanged(const QStringList & paramChanged) { //Selective update (to not update all objects for a simple camera's parameter modification) bool detectorDescriptorParamsChanged = false; bool nearestNeighborParamsChanged = false; for(QStringList::const_iterator iter = paramChanged.begin(); iter!=paramChanged.end(); ++iter) { UINFO("Parameter changed: %s -> \"%s\"", iter->toStdString().c_str(), Settings::getParameter(*iter).toString().toStdString().c_str()); //UINFO("lastObjectsUpdateParameters_.value(*iter)=%s, Settings::getParameter(*iter)=%s", // lastObjectsUpdateParameters_.value(*iter).toString().toStdString().c_str(), // Settings::getParameter(*iter).toString().toStdString().c_str()); if(lastObjectsUpdateParameters_.value(*iter) != Settings::getParameter(*iter)) { if(!detectorDescriptorParamsChanged && iter->contains("Feature2D")) { detectorDescriptorParamsChanged = true; } else if(!nearestNeighborParamsChanged && ( (iter->contains("NearestNeighbor") && Settings::getGeneral_invertedSearch()) || iter->compare(Settings::kGeneral_invertedSearch()) == 0 || (iter->compare(Settings::kGeneral_vocabularyIncremental()) == 0 && Settings::getGeneral_invertedSearch()) || (iter->compare(Settings::kGeneral_threads()) == 0 && !Settings::getGeneral_invertedSearch()) )) { nearestNeighborParamsChanged = true; } lastObjectsUpdateParameters_[*iter] = Settings::getParameter(*iter); } if(iter->compare(Settings::kGeneral_port()) == 0 && Settings::getGeneral_port() != ui_->label_port->text().toInt() && Settings::getGeneral_port() != 0) { setupTCPServer(); } } if(detectorDescriptorParamsChanged) { //Re-init detector and extractor findObject_->updateDetectorExtractor(); } if(Settings::getGeneral_autoUpdateObjects()) { if(detectorDescriptorParamsChanged) { this->updateObjects(); } else if(nearestNeighborParamsChanged) { this->updateVocabulary(); } } else if(objWidgets_.size() && (detectorDescriptorParamsChanged || nearestNeighborParamsChanged)) { this->statusBar()->showMessage(tr("A parameter has changed... \"Update objects\" may be required.")); } if(!camera_->isRunning() && !sceneImage_.empty()) { this->update(); ui_->label_timeRefreshRate->setVisible(false); } ui_->actionCamera_from_video_file->setChecked(!Settings::getCamera_5mediaPath().isEmpty() && !UDirectory::exists(Settings::getCamera_5mediaPath().toStdString())); ui_->actionCamera_from_directory_of_images->setChecked(!Settings::getCamera_5mediaPath().isEmpty() && UDirectory::exists(Settings::getCamera_5mediaPath().toStdString())); ui_->actionCamera_from_TCP_IP->setChecked(Settings::getCamera_6useTcpCamera()); }