diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ccac7ac..d49652a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,9 @@ ELSE () ENDIF() #ADD_DEFINITIONS("-DUNICODE") # to test with UNICODE projects +####### local cmake modules ####### +SET(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake_modules") + ####################### # VERSION ####################### @@ -78,7 +81,8 @@ set(INSTALL_CMAKE_DIR ${DEF_INSTALL_CMAKE_DIR} CACHE PATH ####### DEPENDENCIES ####### FIND_PACKAGE(OpenCV REQUIRED) # tested on 2.3.1 -FIND_PACKAGE(Qt4 COMPONENTS QtCore QtGui QtNetwork) # tested on Qt4.7 +FIND_PACKAGE(Qt4 COMPONENTS QtCore QtGui QtNetwork REQUIRED) # tested on Qt4.8 +FIND_PACKAGE(JSONCPP) ADD_DEFINITIONS(-DQT_NO_KEYWORDS) # To avoid conflicts with boost signals used in ROS ####### OSX BUNDLE CMAKE_INSTALL_PREFIX ####### @@ -103,9 +107,7 @@ ENDIF(APPLE AND BUILD_AS_BUNDLE) ADD_SUBDIRECTORY( src ) ADD_SUBDIRECTORY( app ) ADD_SUBDIRECTORY( example ) -ADD_SUBDIRECTORY( tcpClient ) -ADD_SUBDIRECTORY( imagesTcpServer ) -ADD_SUBDIRECTORY( console_app ) +ADD_SUBDIRECTORY( tools ) @@ -221,4 +223,9 @@ MESSAGE(STATUS " CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}") IF(APPLE) MESSAGE(STATUS " BUILD_AS_BUNDLE = ${BUILD_AS_BUNDLE}") ENDIF(APPLE) +IF(JSONCPP_FOUND) +MESSAGE(STATUS " With JSONCPP = YES") +ELSE() +MESSAGE(STATUS " With JSONCPP = NO (libjsoncpp not found)") +ENDIF() MESSAGE(STATUS "--------------------------------------------") diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index e8de3fc0..149f201f 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -12,6 +12,18 @@ SET(LIBRARIES ${OpenCV_LIBS} ) +IF(JSONCPP_FOUND) + SET(INCLUDE_DIRS + ${INCLUDE_DIRS} + ${JSONCPP_INCLUDE_DIRS} + ) + SET(LIBRARIES + ${LIBRARIES} + ${JSONCPP_LIBRARIES} + ) + ADD_DEFINITIONS("-DWITH_JSONCPP") +ENDIF(JSONCPP_FOUND) + #include files INCLUDE_DIRECTORIES(${INCLUDE_DIRS}) diff --git a/app/main.cpp b/app/main.cpp index 4c870cf8..1a166fe8 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -8,6 +8,10 @@ #include "find_object/TcpServer.h" #include "find_object/utilite/ULogger.h" +#ifdef WITH_JSONCPP +#include +#endif + bool running = true; #ifdef WIN32 @@ -66,15 +70,97 @@ void showUsage() " find_object [options]\n" #endif "Options:\n" - " -console Don't use the GUI (by default the camera will be\n" - " started automatically). Option -objs must also be\n" + " --console Don't use the GUI (by default the camera will be\n" + " started automatically). Option --objects must also be\n" " used with valid objects.\n" - " -objs \"path\" Directory of the objects to detect.\n" - " -config \"path\" Path to configuration file (default: %s).\n" - " -help or --help Show usage.\n", Settings::iniDefaultPath().toStdString().c_str()); + " --objects \"path\" Directory of the objects to detect.\n" + " --config \"path\" Path to configuration file (default: %s).\n" + " --scene \"path\" Path to a scene image file.\n" +#ifdef WITH_JSONCPP + " --json \"path\" Path to an output JSON file (only in --console mode with --scene).\n" +#endif + " --help Show usage.\n", Settings::iniDefaultPath().toStdString().c_str()); exit(-1); } +void writeJSON(const FindObject & findObject, const QString & path) +{ +#ifdef WITH_JSONCPP + if(!path.isEmpty()) + { + Json::Value root; + Json::Value detections; + Json::Value matchesValues; + + if(findObject.objectsDetected().size()) + { + Q_ASSERT(objectsDetected.size() == findObject.inliers().size() && + objectsDetected.size() == findObject.outliers().size()); + + const QMultiMap > & objectsDetected = findObject.objectsDetected(); + QMultiMap >::const_iterator iterInliers = findObject.inliers().constBegin(); + QMultiMap >::const_iterator iterOutliers = findObject.outliers().constBegin(); + for(QMultiMap >::const_iterator iter = objectsDetected.constBegin(); + iter!= objectsDetected.end();) + { + char index = 'a'; + QMultiMap >::const_iterator jter = iter; + for(;jter != objectsDetected.constEnd() && jter.key() == iter.key(); ++jter) + { + QString name = QString("object_%1%2").arg(jter.key()).arg(objectsDetected.count(jter.key())>1?QString(index++):""); + detections.append(name.toStdString()); + + Json::Value homography; + homography.append(jter.value().second.m11()); + homography.append(jter.value().second.m12()); + homography.append(jter.value().second.m13()); + homography.append(jter.value().second.m21()); + homography.append(jter.value().second.m22()); + homography.append(jter.value().second.m23()); + homography.append(jter.value().second.m31()); // dx + homography.append(jter.value().second.m32()); // dy + homography.append(jter.value().second.m33()); + root[name.toStdString()]["width"] = jter.value().first.width(); + root[name.toStdString()]["height"] = jter.value().first.height(); + root[name.toStdString()]["homography"] = homography; + root[name.toStdString()]["inliers"] = iterInliers.value().size(); + root[name.toStdString()]["outliers"] = iterOutliers.value().size(); + + ++iterInliers; + ++iterOutliers; + } + iter = jter; + } + } + + const QMap > & matches = findObject.matches(); + for(QMap >::const_iterator iter = matches.constBegin(); + iter != matches.end(); + ++iter) + { + QString name = QString("matches_%1").arg(iter.key()); + root[name.toStdString()] = iter.value().size(); + matchesValues.append(name.toStdString()); + } + + root["objects"] = detections; + root["matches"] = matchesValues; + + // write in a nice readible way + Json::StyledWriter styledWriter; + //std::cout << styledWriter.write(root); + QFile file(path); + file.open(QIODevice::WriteOnly | QIODevice::Text); + QTextStream out(&file); + out << styledWriter.write(root).c_str(); + file.close(); + UINFO("JSON written to \"%s\"", path.toStdString().c_str()); + } +#else + UERROR("Not built with JSON support!"); +#endif +} + int main(int argc, char* argv[]) { ULogger::setType(ULogger::kTypeConsole); @@ -87,11 +173,16 @@ int main(int argc, char* argv[]) ////////////////////////// bool guiMode = true; QString objectsPath = ""; + QString scenePath = ""; QString configPath = Settings::iniDefaultPath(); + QString jsonPath; for(int i=1; i [FindObject] ---ObjectsDetected---> [TcpServer] - QObject::connect(&camera, SIGNAL(imageReceived(const cv::Mat &)), findObject, SLOT(detect(const cv::Mat &))); + // [FindObject] ---ObjectsDetected---> [TcpServer] QObject::connect(findObject, SIGNAL(objectsFound(QMultiMap >)), &tcpServer, SLOT(publishObjects(QMultiMap >))); - setupQuitSignal(); - - // start processing! - while(running && !camera.start()) + if(!scene.empty()) { - if(Settings::getCamera_6useTcpCamera()) + // process the scene and exit + findObject->detect(scene); // this will automatically emit objectsFound() + + if(!jsonPath.isEmpty()) { - UWARN("Camera initialization failed! (with server %s:%d) Trying again in 1 second...", - Settings::getCamera_7IP().toStdString().c_str(), Settings::getCamera_8port()); - Sleep(1000); - } - else - { - UERROR("Camera initialization failed!"); - running = false; + writeJSON(*findObject, jsonPath); } } - if(running) + else { - app.exec(); + Camera camera; + + // [Camera] ---Image---> [FindObject] + QObject::connect(&camera, SIGNAL(imageReceived(const cv::Mat &)), findObject, SLOT(detect(const cv::Mat &))); + + //use camera in settings + setupQuitSignal(); + + // start processing! + while(running && !camera.start()) + { + if(Settings::getCamera_6useTcpCamera()) + { + UWARN("Camera initialization failed! (with server %s:%d) Trying again in 1 second...", + Settings::getCamera_7IP().toStdString().c_str(), Settings::getCamera_8port()); + Sleep(1000); + } + else + { + UERROR("Camera initialization failed!"); + running = false; + } + } + if(running) + { + app.exec(); + } + + // cleanup + camera.stop(); } - // cleanup - camera.stop(); delete findObject; tcpServer.close(); } - - // Save settings - Settings::saveSettings(); } diff --git a/cmake_modules/FindJSONCPP.cmake b/cmake_modules/FindJSONCPP.cmake new file mode 100644 index 00000000..64fe7258 --- /dev/null +++ b/cmake_modules/FindJSONCPP.cmake @@ -0,0 +1,29 @@ +# - Find JSONCPP +# This module finds an installed JSONCPP package. +# +# It sets the following variables: +# JSONCPP_FOUND - Set to false, or undefined, if JSONCPP isn't found. +# JSONCPP_INCLUDE_DIRS - The JSONCPP include directory. +# JSONCPP_LIBRARIES - The JSONCPP library to link against. + +FIND_PATH(JSONCPP_INCLUDE_DIRS json/features.h PATH_SUFFIXES jsoncpp) + +FIND_LIBRARY(JSONCPP_LIBRARY NAMES jsoncpp) + +IF (JSONCPP_INCLUDE_DIRS AND JSONCPP_LIBRARY) + SET(JSONCPP_FOUND TRUE) +ENDIF (JSONCPP_INCLUDE_DIRS AND JSONCPP_LIBRARY) + +IF (JSONCPP_FOUND) + # show which JSONCPP was found only if not quiet + SET(JSONCPP_LIBRARIES ${JSONCPP_LIBRARY}) + IF (NOT JSONCPP_FIND_QUIETLY) + MESSAGE(STATUS "Found JSONCPP: ${JSONCPP_LIBRARIES}") + ENDIF (NOT JSONCPP_FIND_QUIETLY) +ELSE (JSONCPP_FOUND) + # fatal error if JSONCPP is required but not found + IF (JSONCPP_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Could not find JSONCPP (libjsoncpp)") + ENDIF (JSONCPP_FIND_REQUIRED) +ENDIF (JSONCPP_FOUND) + diff --git a/include/find_object/FindObject.h b/include/find_object/FindObject.h index 25f81ab4..e4f2ee25 100644 --- a/include/find_object/FindObject.h +++ b/include/find_object/FindObject.h @@ -53,6 +53,7 @@ public: const QMap & objects() const {return objects_;} const Vocabulary * vocabulary() const {return vocabulary_;} + const QMultiMap > & objectsDetected() const {return objectsDetected_;} const QMap & timeStamps() const {return timeStamps_;} const std::vector & sceneKeypoints() const {return sceneKeypoints_;} const cv::Mat & sceneDescriptors() const {return sceneDescriptors_;} @@ -66,7 +67,7 @@ public: float maxMatchedDistance() const {return maxMatchedDistance_;} public Q_SLOTS: - void detect(const cv::Mat & image); + void detect(const cv::Mat & image); // emit objectsfound() Q_SIGNALS: void objectsFound(const QMultiMap > &); @@ -82,6 +83,7 @@ private: KeypointDetector * detector_; DescriptorExtractor * extractor_; + QMultiMap > objectsDetected_; QMap timeStamps_; std::vector sceneKeypoints_; cv::Mat sceneDescriptors_; diff --git a/include/find_object/MainWindow.h b/include/find_object/MainWindow.h index 9864a75b..f23196f1 100644 --- a/include/find_object/MainWindow.h +++ b/include/find_object/MainWindow.h @@ -50,6 +50,7 @@ public Q_SLOTS: void startProcessing(); void stopProcessing(); void pauseProcessing(); + void update(const cv::Mat & image = cv::Mat()); private Q_SLOTS: void loadSettings(); @@ -67,7 +68,6 @@ private Q_SLOTS: void updateObjectsSize(); void updateMirrorView(); void showHideControls(); - void update(const cv::Mat & image = cv::Mat()); void updateObjects(); void notifyParametersChanged(const QStringList & param); void moveCameraFrame(int frame); diff --git a/src/FindObject.cpp b/src/FindObject.cpp index d8f97210..148b43f1 100644 --- a/src/FindObject.cpp +++ b/src/FindObject.cpp @@ -594,6 +594,8 @@ private: void FindObject::detect(const cv::Mat & image) { + QTime time; + time.start(); QMultiMap > objects; this->detect(image, objects); if(objects.size() > 0 || Settings::getGeneral_sendNoObjDetectedEvents()) @@ -603,20 +605,23 @@ void FindObject::detect(const cv::Mat & image) if(objects.size() > 1) { - UINFO("(%s) %d objects detected!", + UINFO("(%s) %d objects detected! (%d ms)", QTime::currentTime().toString("HH:mm:ss.zzz").toStdString().c_str(), - (int)objects.size()); + (int)objects.size(), + time.elapsed()); } else if(objects.size() == 1) { - UINFO("(%s) Object %d detected!", + UINFO("(%s) Object %d detected! (%d ms)", QTime::currentTime().toString("HH:mm:ss.zzz").toStdString().c_str(), - (int)objects.begin().key()); + (int)objects.begin().key(), + time.elapsed()); } else if(Settings::getGeneral_sendNoObjDetectedEvents()) { - UINFO("(%s) No objects detected.", - QTime::currentTime().toString("HH:mm:ss.zzz").toStdString().c_str()); + UINFO("(%s) No objects detected. (%d ms)", + QTime::currentTime().toString("HH:mm:ss.zzz").toStdString().c_str(), + time.elapsed()); } } @@ -626,6 +631,7 @@ bool FindObject::detect(const cv::Mat & image, QMultiMap +#include + +// OpenCV stuff +#include +#include +#include +#include +#include // for homography + +void showUsage() +{ + printf( + "\n" + "Return similarity between two images (the number of similar features between the images).\n" + "Usage :\n" + " ./find_object-similarity [option] object.png scene.png\n" + "Options: \n" + " -inliers return inliers percentage : inliers / (inliers + outliers)\n" + " -quiet don't show messages\n"); + + exit(-1); +} + +enum {mTotal, mInliers}; + +int main(int argc, char * argv[]) +{ + bool quiet = false; + int method = mTotal; //total matches + if(argc<3) + { + printf("Two images required!\n"); + showUsage(); + } + else if(argc>3) + { + for(int i=1; i objectKeypoints; + std::vector sceneKeypoints; + cv::Mat objectDescriptors; + cv::Mat sceneDescriptors; + + //////////////////////////// + // EXTRACT KEYPOINTS + //////////////////////////// + cv::SIFT sift; + sift.detect(objectImg, objectKeypoints); + sift.detect(sceneImg, sceneKeypoints); + + //////////////////////////// + // EXTRACT DESCRIPTORS + //////////////////////////// + sift.compute(objectImg, objectKeypoints, objectDescriptors); + sift.compute(sceneImg, sceneKeypoints, sceneDescriptors); + + //////////////////////////// + // NEAREST NEIGHBOR MATCHING USING FLANN LIBRARY (included in OpenCV) + //////////////////////////// + cv::Mat results; + cv::Mat dists; + std::vector > matches; + int k=2; // find the 2 nearest neighbors + + // Create Flann KDTree index + cv::flann::Index flannIndex(sceneDescriptors, cv::flann::KDTreeIndexParams(), cvflann::FLANN_DIST_EUCLIDEAN); + results = cv::Mat(objectDescriptors.rows, k, CV_32SC1); // Results index + dists = cv::Mat(objectDescriptors.rows, k, CV_32FC1); // Distance results are CV_32FC1 + + // search (nearest neighbor) + flannIndex.knnSearch(objectDescriptors, results, dists, k, cv::flann::SearchParams() ); + + //////////////////////////// + // PROCESS NEAREST NEIGHBOR RESULTS + //////////////////////////// + + // Find correspondences by NNDR (Nearest Neighbor Distance Ratio) + float nndrRatio = 0.6; + std::vector mpts_1, mpts_2; // Used for homography + std::vector indexes_1, indexes_2; // Used for homography + std::vector outlier_mask; // Used for homography + // Check if this descriptor matches with those of the objects + + for(int i=0; i(i,0) <= nndrRatio * dists.at(i,1)) + { + mpts_1.push_back(objectKeypoints.at(i).pt); + indexes_1.push_back(i); + + mpts_2.push_back(sceneKeypoints.at(results.at(i,0)).pt); + indexes_2.push_back(results.at(i,0)); + } + } + + if(method == mInliers) + { + // FIND HOMOGRAPHY + unsigned int minInliers = 8; + if(mpts_1.size() >= minInliers) + { + cv::Mat H = findHomography(mpts_1, + mpts_2, + cv::RANSAC, + 1.0, + outlier_mask); + int inliers=0, outliers=0; + for(unsigned int k=0; k