Added Feature SubPixel refining and OpticalFlow homography refining

This commit is contained in:
matlabbe 2015-05-19 21:23:24 -04:00
parent b3b25b905b
commit 4b2ab25cb7
5 changed files with 241 additions and 121 deletions

View File

@ -42,6 +42,7 @@ public:
enum TimeStamp{
kTimeKeypointDetection,
kTimeDescriptorExtraction,
kTimeSubPixelRefining,
kTimeSkewAffine,
kTimeIndexing,
kTimeMatching,

View File

@ -120,6 +120,10 @@ class FINDOBJECT_EXP Settings
PARAMETER(Feature2D, 3MaxFeatures, int, 0, "Maximum features per image. If the number of features extracted is over this threshold, only X features with the highest response are kept. 0 means all features are kept.");
PARAMETER(Feature2D, 4Affine, bool, false, "(ASIFT) Extract features on multiple affine transformations of the image.");
PARAMETER(Feature2D, 5AffineCount, int, 6, "(ASIFT) Higher the value, more affine transformations will be done.");
PARAMETER(Feature2D, 6SubPix, bool, FINDOBJECT_NONFREE != 1, "Refines the corner locations. With SIFT/SURF, features are already subpixel, so no need to activate this.");
PARAMETER(Feature2D, 7SubPixWinSize, int, 3, "Half of the side length of the search window. For example, if winSize=Size(5,5) , then a 5*2+1 x 5*2+1 = 11 x 11 search window is used.");
PARAMETER(Feature2D, 8SubPixIterations, int, 30, "The process of corner position refinement stops after X iterations.");
PARAMETER(Feature2D, 9SubPixEps, float, 0.02, "The process of corner position refinement stops when the corner position moves by less than epsilon on some iteration.");
PARAMETER(Feature2D, Brief_bytes, int, 32, "Bytes is a length of descriptor in bytes. It can be equal 16, 32 or 64 bytes.");
@ -255,6 +259,11 @@ class FINDOBJECT_EXP Settings
PARAMETER(Homography, rectBorderWidth, int, 4, "Homography rectangle border width.");
PARAMETER(Homography, allCornersVisible, bool, false, "All corners of the detected object must be visible in the scene.");
PARAMETER(Homography, minAngle, int, 0, "(Degrees) Homography minimum angle. Set 0 to disable. When the angle is very small, this is a good indication that the homography is wrong. A good value is over 60 degrees.");
PARAMETER(Homography, opticalFlow, bool, false, "Activate optical flow to refine matched features before computing the homography.");
PARAMETER(Homography, opticalFlowWinSize, int, 16, "Size of the search window at each pyramid level.");
PARAMETER(Homography, opticalFlowMaxLevel, int, 3, "0-based maximal pyramid level number; if set to 0, pyramids are not used (single level), if set to 1, two levels are used, and so on; if pyramids are passed to input then algorithm will use as many levels as pyramids have but no more than maxLevel.");
PARAMETER(Homography, opticalFlowIterations, int, 30, "Specifying the termination criteria of the iterative search algorithm (after the specified maximum number of iterations).");
PARAMETER(Homography, opticalFlowEps, float, 0.01, "Specifying the termination criteria of the iterative search algorithm (when the search window moves by less than epsilon).");
public:
virtual ~Settings(){}

View File

@ -377,7 +377,8 @@ public:
phi_(phi),
timeSkewAffine_(0),
timeDetection_(0),
timeExtraction_(0)
timeExtraction_(0),
timeSubPix_(0)
{
UASSERT(detector && extractor);
}
@ -388,6 +389,7 @@ public:
int timeSkewAffine() const {return timeSkewAffine_;}
int timeDetection() const {return timeDetection_;}
int timeExtraction() const {return timeExtraction_;}
int timeSubPix() const {return timeSubPix_;}
protected:
virtual void run()
@ -422,6 +424,24 @@ protected:
keypoints_[i].pt.x = pa.at<float>(0,0);
keypoints_[i].pt.y = pa.at<float>(1,0);
}
if(keypoints_.size() && Settings::getFeature2D_6SubPix())
{
// Sub pixel should be done after descriptors extraction
std::vector<cv::Point2f> corners;
cv::KeyPoint::convert(keypoints_, corners);
cv::cornerSubPix(image_,
corners,
cv::Size(Settings::getFeature2D_7SubPixWinSize(), Settings::getFeature2D_7SubPixWinSize()),
cv::Size(-1,-1),
cv::TermCriteria( CV_TERMCRIT_EPS + CV_TERMCRIT_ITER, Settings::getFeature2D_8SubPixIterations(), Settings::getFeature2D_9SubPixEps() ));
UASSERT(corners.size() == keypoints_.size());
for(unsigned int i=0; i<corners.size(); ++i)
{
keypoints_[i].pt = corners[i];
}
timeSubPix_ +=timeStep.restart();
}
}
else
{
@ -440,6 +460,7 @@ private:
int timeSkewAffine_;
int timeDetection_;
int timeExtraction_;
int timeSubPix_;
};
class ExtractFeaturesThread : public QThread
@ -456,7 +477,8 @@ public:
image_(image),
timeSkewAffine_(0),
timeDetection_(0),
timeExtraction_(0)
timeExtraction_(0),
timeSubPix_(0)
{
UASSERT(detector && extractor);
}
@ -468,6 +490,7 @@ public:
int timeSkewAffine() const {return timeSkewAffine_;}
int timeDetection() const {return timeDetection_;}
int timeExtraction() const {return timeExtraction_;}
int timeSubPix() const {return timeSubPix_;}
protected:
virtual void run()
@ -513,6 +536,23 @@ protected:
{
UERROR("obj=%d kpt=%d != descriptors=%d", objectId_, (int)keypoints_.size(), descriptors_.rows);
}
else if(Settings::getFeature2D_6SubPix())
{
// Sub pixel should be done after descriptors extraction
std::vector<cv::Point2f> corners;
cv::KeyPoint::convert(keypoints_, corners);
cv::cornerSubPix(image_,
corners,
cv::Size(Settings::getFeature2D_7SubPixWinSize(), Settings::getFeature2D_7SubPixWinSize()),
cv::Size(-1,-1),
cv::TermCriteria( CV_TERMCRIT_EPS + CV_TERMCRIT_ITER, Settings::getFeature2D_8SubPixIterations(), Settings::getFeature2D_9SubPixEps() ));
UASSERT(corners.size() == keypoints_.size());
for(unsigned int i=0; i<corners.size(); ++i)
{
keypoints_[i].pt = corners[i];
}
timeSubPix_ +=timeStep.restart();
}
}
else
{
@ -566,6 +606,7 @@ protected:
timeSkewAffine_ += threads[k]->timeSkewAffine();
timeDetection_ += threads[k]->timeDetection();
timeExtraction_ += threads[k]->timeExtraction();
timeSubPix_ += threads[k]->timeSubPix();
}
}
}
@ -583,6 +624,7 @@ private:
int timeSkewAffine_;
int timeDetection_;
int timeExtraction_;
int timeSubPix_;
};
void FindObject::updateObjects(const QList<int> & ids)
@ -876,11 +918,15 @@ public:
const QMultiMap<int, int> * matches, // <object, scene>
int objectId,
const std::vector<cv::KeyPoint> * kptsA,
const std::vector<cv::KeyPoint> * kptsB) :
const std::vector<cv::KeyPoint> * kptsB,
const cv::Mat & imageA, // image only required if opticalFlow is on
const cv::Mat & imageB) : // image only required if opticalFlow is on
matches_(matches),
objectId_(objectId),
kptsA_(kptsA),
kptsB_(kptsB),
imageA_(imageA),
imageB_(imageB),
code_(DetectionInfo::kRejectedUndef)
{
UASSERT(matches && kptsA && kptsB);
@ -919,6 +965,41 @@ protected:
if((int)mpts_1.size() >= Settings::getHomography_minimumInliers())
{
if(Settings::getHomography_opticalFlow())
{
UASSERT(!imageA_.empty() && !imageB_.empty());
cv::Mat imageA = imageA_;
cv::Mat imageB = imageB_;
if(imageA_.cols < imageB_.cols && imageA_.rows < imageB_.rows)
{
// padding, optical flow wants images of the same size
imageA = cv::Mat::zeros(imageB_.size(), imageA_.type());
imageA_.copyTo(imageA(cv::Rect(0,0,imageA_.cols, imageA_.rows)));
}
if(imageA.size() == imageB.size())
{
//refine matches
std::vector<unsigned char> status;
std::vector<float> err;
cv::calcOpticalFlowPyrLK(
imageA,
imageB_,
mpts_1,
mpts_2,
status,
err,
cv::Size(Settings::getHomography_opticalFlowWinSize(), Settings::getHomography_opticalFlowWinSize()),
Settings::getHomography_opticalFlowMaxLevel(),
cv::TermCriteria(cv::TermCriteria::COUNT+cv::TermCriteria::EPS, Settings::getHomography_opticalFlowIterations(), Settings::getHomography_opticalFlowEps()),
cv::OPTFLOW_LK_GET_MIN_EIGENVALS | cv::OPTFLOW_USE_INITIAL_FLOW, 1e-4);
}
else
{
UERROR("Object's image should be less/equal size of the scene image to use Optical Flow.");
}
}
h_ = findHomography(mpts_1,
mpts_2,
Settings::getHomographyMethod(),
@ -959,6 +1040,8 @@ private:
int objectId_;
const std::vector<cv::KeyPoint> * kptsA_;
const std::vector<cv::KeyPoint> * kptsB_;
cv::Mat imageA_;
cv::Mat imageB_;
DetectionInfo::RejectedCode code_;
std::vector<int> indexesA_;
@ -1033,6 +1116,7 @@ bool FindObject::detect(const cv::Mat & image, find_object::DetectionInfo & info
info.sceneDescriptors_ = extractThread.descriptors();
info.timeStamps_.insert(DetectionInfo::kTimeKeypointDetection, extractThread.timeDetection());
info.timeStamps_.insert(DetectionInfo::kTimeDescriptorExtraction, extractThread.timeExtraction());
info.timeStamps_.insert(DetectionInfo::kTimeSubPixelRefining, extractThread.timeSubPix());
info.timeStamps_.insert(DetectionInfo::kTimeSkewAffine, extractThread.timeSkewAffine());
bool consistentNNData = (vocabulary_->size()!=0 && vocabulary_->wordToObjects().begin().value()!=-1 && Settings::getGeneral_invertedSearch()) ||
@ -1063,9 +1147,9 @@ bool FindObject::detect(const cv::Mat & image, find_object::DetectionInfo & info
if(!Settings::getGeneral_invertedSearch())
{
vocabulary_->clear();
// CREATE INDEX for the scene
UDEBUG("CREATE INDEX FOR THE SCENE");
vocabulary_->clear();
words = vocabulary_->addWords(info.sceneDescriptors_, -1, Settings::getGeneral_vocabularyIncremental());
if(!Settings::getGeneral_vocabularyIncremental())
{
@ -1235,7 +1319,13 @@ bool FindObject::detect(const cv::Mat & image, find_object::DetectionInfo & info
for(int k=i; k<i+threadCounts && k<matchesList.size(); ++k)
{
int objectId = matchesId[k];
threads.push_back(new HomographyThread(&matchesList[k], objectId, &objects_.value(objectId)->keypoints(), &info.sceneKeypoints_));
threads.push_back(new HomographyThread(
&matchesList[k],
objectId,
&objects_.value(objectId)->keypoints(),
&info.sceneKeypoints_,
objects_.value(objectId)->image(),
grayscaleImg));
threads.back()->start();
}
@ -1353,8 +1443,6 @@ bool FindObject::detect(const cv::Mat & image, find_object::DetectionInfo & info
if(code == DetectionInfo::kRejectedUndef)
{
// Accepted!
//std::cout << "H= " << threads[j]->getHomography() << std::endl;
info.objDetected_.insert(id, hTransform);
info.objDetectedSizes_.insert(id, objects_.value(id)->rect().size());
info.objDetectedInliers_.insert(id, threads[j]->getInliers());

View File

@ -159,7 +159,6 @@ MainWindow::MainWindow(FindObject * findObject, Camera * camera, QWidget * paren
ui_->menuView->addAction(ui_->dockWidget_plot->toggleViewAction());
connect(ui_->toolBox, SIGNAL(parametersChanged(const QStringList &)), this, SLOT(notifyParametersChanged(const QStringList &)));
ui_->imageView_source->setGraphicsViewMode(true);
ui_->imageView_source->setTextLabel(tr("Press \"space\" to start the camera or drop an image here..."));
ui_->imageView_source->setMirrorView(Settings::getGeneral_mirrorView());
connect((QCheckBox*)ui_->toolBox->getParameterWidget(Settings::kGeneral_mirrorView()),
@ -1139,6 +1138,7 @@ void MainWindow::update(const cv::Mat & image)
ui_->label_timeDetection->setNum(info.timeStamps_.value(DetectionInfo::kTimeKeypointDetection, 0));
ui_->label_timeSkewAffine->setNum(info.timeStamps_.value(DetectionInfo::kTimeSkewAffine, 0));
ui_->label_timeExtraction->setNum(info.timeStamps_.value(DetectionInfo::kTimeDescriptorExtraction, 0));
ui_->label_timeSubPix->setNum(info.timeStamps_.value(DetectionInfo::kTimeSubPixelRefining, 0));
ui_->imageView_source->setData(info.sceneKeypoints_, cvtCvMat2QImage(sceneImage_));
if(!findObject_->vocabulary()->size())
{
@ -1339,6 +1339,7 @@ void MainWindow::update(const cv::Mat & image)
ui_->label_timeDetection->setNum(info.timeStamps_.value(DetectionInfo::kTimeKeypointDetection, 0));
ui_->label_timeSkewAffine->setNum(info.timeStamps_.value(DetectionInfo::kTimeSkewAffine, 0));
ui_->label_timeExtraction->setNum(info.timeStamps_.value(DetectionInfo::kTimeDescriptorExtraction, 0));
ui_->label_timeSubPix->setNum(info.timeStamps_.value(DetectionInfo::kTimeSubPixelRefining, 0));
ui_->imageView_source->setData(info.sceneKeypoints_, cvtCvMat2QImage(sceneImage_));
}

View File

@ -7,7 +7,7 @@
<x>0</x>
<y>0</y>
<width>881</width>
<height>536</height>
<height>552</height>
</rect>
</property>
<property name="windowTitle">
@ -368,7 +368,7 @@
<x>0</x>
<y>0</y>
<width>222</width>
<height>409</height>
<height>425</height>
</rect>
</property>
<layout class="QVBoxLayout" name="verticalLayout_objects">
@ -475,84 +475,175 @@
<property name="verticalSpacing">
<number>0</number>
</property>
<item row="4" column="1">
<item row="5" column="1">
<widget class="QLabel" name="label_timeIndexing">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="5" column="1">
<item row="6" column="1">
<widget class="QLabel" name="label_timeMatching">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="4" column="2">
<item row="5" column="2">
<widget class="QLabel" name="label_9">
<property name="text">
<string>ms</string>
</property>
</widget>
</item>
<item row="5" column="2">
<item row="6" column="2">
<widget class="QLabel" name="label_10">
<property name="text">
<string>ms</string>
</property>
</widget>
</item>
<item row="4" column="0">
<item row="0" column="2">
<widget class="QLabel" name="label_16">
<property name="text">
<string>ms</string>
</property>
</widget>
</item>
<item row="12" column="0">
<widget class="QLabel" name="label_17">
<property name="text">
<string>Vocabulary size</string>
</property>
</widget>
</item>
<item row="12" column="1">
<widget class="QLabel" name="label_vocabularySize">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="13" column="0">
<widget class="QLabel" name="label_18">
<property name="text">
<string>IP address</string>
</property>
</widget>
</item>
<item row="14" column="0">
<widget class="QLabel" name="label_19">
<property name="text">
<string>Output detection port</string>
</property>
</widget>
</item>
<item row="13" column="1">
<widget class="QLabel" name="label_ipAddress">
<property name="text">
<string>0.0.0.0</string>
</property>
</widget>
</item>
<item row="14" column="1">
<widget class="QLabel" name="label_port">
<property name="text">
<string>0</string>
</property>
</widget>
</item>
<item row="9" column="0">
<widget class="QLabel" name="label_20">
<property name="text">
<string>Objects detected</string>
</property>
</widget>
</item>
<item row="9" column="1">
<widget class="QLabel" name="label_objectsDetected">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="7" column="1">
<widget class="QLabel" name="label_timeHomographies">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="15" column="0">
<widget class="QLabel" name="label_21">
<property name="text">
<string>Input image port</string>
</property>
</widget>
</item>
<item row="15" column="1">
<widget class="QLabel" name="label_port_image">
<property name="text">
<string>-</string>
</property>
</widget>
</item>
<item row="3" column="0">
<widget class="QLabel" name="label_22">
<property name="text">
<string>Affine transforms</string>
</property>
</widget>
</item>
<item row="5" column="0">
<widget class="QLabel" name="label_8">
<property name="text">
<string>Descriptors indexing</string>
</property>
</widget>
</item>
<item row="6" column="2">
<item row="7" column="2">
<widget class="QLabel" name="label_12">
<property name="text">
<string>ms</string>
</property>
</widget>
</item>
<item row="9" column="1">
<item row="10" column="1">
<widget class="QLabel" name="label_minMatchedDistance">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="9" column="0">
<item row="10" column="0">
<widget class="QLabel" name="label_13">
<property name="text">
<string>Min matched distance</string>
</property>
</widget>
</item>
<item row="10" column="0">
<item row="11" column="0">
<widget class="QLabel" name="label_14">
<property name="text">
<string>Max matched distance</string>
</property>
</widget>
</item>
<item row="10" column="1">
<item row="11" column="1">
<widget class="QLabel" name="label_maxMatchedDistance">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="6" column="0">
<item row="7" column="0">
<widget class="QLabel" name="label_11">
<property name="text">
<string>Homograhies</string>
</property>
</widget>
</item>
<item row="3" column="0">
<item row="4" column="0">
<widget class="QLabel" name="label_2">
<property name="text">
<string>Descriptors extraction</string>
@ -566,14 +657,14 @@
</property>
</widget>
</item>
<item row="3" column="1">
<item row="4" column="1">
<widget class="QLabel" name="label_timeExtraction">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="3" column="2">
<item row="4" column="2">
<widget class="QLabel" name="label_4">
<property name="text">
<string>ms</string>
@ -594,7 +685,7 @@
</property>
</widget>
</item>
<item row="5" column="0">
<item row="6" column="0">
<widget class="QLabel" name="label_7">
<property name="text">
<string>Descriptors matching</string>
@ -615,132 +706,62 @@
</property>
</widget>
</item>
<item row="0" column="2">
<widget class="QLabel" name="label_16">
<property name="text">
<string>ms</string>
</property>
</widget>
</item>
<item row="11" column="0">
<widget class="QLabel" name="label_17">
<property name="text">
<string>Vocabulary size</string>
</property>
</widget>
</item>
<item row="11" column="1">
<widget class="QLabel" name="label_vocabularySize">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="12" column="0">
<widget class="QLabel" name="label_18">
<property name="text">
<string>IP address</string>
</property>
</widget>
</item>
<item row="13" column="0">
<widget class="QLabel" name="label_19">
<property name="text">
<string>Output detection port</string>
</property>
</widget>
</item>
<item row="12" column="1">
<widget class="QLabel" name="label_ipAddress">
<property name="text">
<string>0.0.0.0</string>
</property>
</widget>
</item>
<item row="13" column="1">
<widget class="QLabel" name="label_port">
<property name="text">
<string>0</string>
</property>
</widget>
</item>
<item row="8" column="0">
<widget class="QLabel" name="label_20">
<property name="text">
<string>Objects detected</string>
</property>
</widget>
</item>
<item row="8" column="1">
<widget class="QLabel" name="label_objectsDetected">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="6" column="1">
<widget class="QLabel" name="label_timeHomographies">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="14" column="0">
<widget class="QLabel" name="label_21">
<property name="text">
<string>Input image port</string>
</property>
</widget>
</item>
<item row="14" column="1">
<widget class="QLabel" name="label_port_image">
<property name="text">
<string>-</string>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QLabel" name="label_22">
<property name="text">
<string>Affine transforms</string>
</property>
</widget>
</item>
<item row="2" column="1">
<item row="3" column="1">
<widget class="QLabel" name="label_timeSkewAffine">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="2" column="2">
<item row="3" column="2">
<widget class="QLabel" name="label_23">
<property name="text">
<string>ms</string>
</property>
</widget>
</item>
<item row="7" column="0">
<item row="8" column="0">
<widget class="QLabel" name="label_24">
<property name="text">
<string>Refresh GUI</string>
</property>
</widget>
</item>
<item row="7" column="1">
<item row="8" column="1">
<widget class="QLabel" name="label_timeRefreshGUI">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="7" column="2">
<item row="8" column="2">
<widget class="QLabel" name="label_25">
<property name="text">
<string>ms</string>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QLabel" name="label_26">
<property name="text">
<string>Features sub pixel refining</string>
</property>
</widget>
</item>
<item row="2" column="1">
<widget class="QLabel" name="label_timeSubPix">
<property name="text">
<string>000</string>
</property>
</widget>
</item>
<item row="2" column="2">
<widget class="QLabel" name="label_27">
<property name="text">
<string>ms</string>
</property>
</widget>
</item>
</layout>
</item>
<item>