Session: Compression of the vocabulary to avoid QByteArray max 2GB limitation

CMakeLists.txt

@@ -64,6 +64,8 @@ ADD_DEFINITIONS(-DQT_NO_KEYWORDS) # To avoid conflicts with boost signals used i
 
 FIND_PACKAGE(Tcmalloc QUIET)
 
+FIND_PACKAGE(ZLIB REQUIRED QUIET)
+
 SET(NONFREE 0)
 IF(OPENCV_NONFREE_FOUND OR OPENCV_XFEATURES2D_FOUND)
    SET(NONFREE 1)

src/CMakeLists.txt

@@ -74,6 +74,7 @@ SET(SRC_FILES
    ./utilite/UConversion.cpp
    ./rtabmap/PdfPlot.cpp
    ./json/jsoncpp.cpp
+   ./Compression.cpp
    ${moc_srcs} 
    ${moc_uis} 
    ${srcs_qrc}
@@ -93,6 +94,7 @@ SET(INCLUDE_DIRS
    ${CMAKE_CURRENT_SOURCE_DIR}
    ${OpenCV_INCLUDE_DIRS}
    ${CMAKE_CURRENT_BINARY_DIR} # for qt ui generated in binary dir
+   ${ZLIB_INCLUDE_DIRS}
 )
 IF(CATKIN_BUILD)
    SET(INCLUDE_DIRS
@@ -108,6 +110,7 @@ ENDIF(QT4_FOUND)
 SET(LIBRARIES
    ${QT_LIBRARIES} 
    ${OpenCV_LIBS} 
+   ${ZLIB_LIBRARIES} 
 )
 IF(CATKIN_BUILD)
    SET(LIBRARIES

src/Compression.cpp

@@ -0,0 +1,79 @@
+/*
+ * Compression.cpp
+ *
+ *  Created on: Sep 10, 2018
+ *      Author: labm2414
+ */
+
+#include <Compression.h>
+#include <zlib.h>
+#include "find_object/utilite/ULogger.h"
+
+namespace find_object {
+
+std::vector<unsigned char> compressData(const cv::Mat & data)
+{
+	std::vector<unsigned char> bytes;
+	if(!data.empty())
+	{
+		uLong sourceLen = uLong(data.total())*uLong(data.elemSize());
+		uLong destLen = compressBound(sourceLen);
+		bytes.resize(destLen);
+		int errCode = compress(
+						(Bytef *)bytes.data(),
+						&destLen,
+						(const Bytef *)data.data,
+						sourceLen);
+
+		bytes.resize(destLen+3*sizeof(int));
+		*((int*)&bytes[destLen]) = data.rows;
+		*((int*)&bytes[destLen+sizeof(int)]) = data.cols;
+		*((int*)&bytes[destLen+2*sizeof(int)]) = data.type();
+
+		if(errCode == Z_MEM_ERROR)
+		{
+			UERROR("Z_MEM_ERROR : Insufficient memory.");
+		}
+		else if(errCode == Z_BUF_ERROR)
+		{
+			UERROR("Z_BUF_ERROR : The buffer dest was not large enough to hold the uncompressed data.");
+		}
+	}
+	return bytes;
+}
+
+cv::Mat uncompressData(const unsigned char * bytes, unsigned long size)
+{
+	cv::Mat data;
+	if(bytes && size>=3*sizeof(int))
+	{
+		//last 3 int elements are matrix size and type
+		int height = *((int*)&bytes[size-3*sizeof(int)]);
+		int width = *((int*)&bytes[size-2*sizeof(int)]);
+		int type = *((int*)&bytes[size-1*sizeof(int)]);
+
+		data = cv::Mat(height, width, type);
+		uLongf totalUncompressed = uLongf(data.total())*uLongf(data.elemSize());
+
+		int errCode = uncompress(
+						(Bytef*)data.data,
+						&totalUncompressed,
+						(const Bytef*)bytes,
+						uLong(size));
+
+		if(errCode == Z_MEM_ERROR)
+		{
+			UERROR("Z_MEM_ERROR : Insufficient memory.");
+		}
+		else if(errCode == Z_BUF_ERROR)
+		{
+			UERROR("Z_BUF_ERROR : The buffer dest was not large enough to hold the uncompressed data.");
+		}
+		else if(errCode == Z_DATA_ERROR)
+		{
+			UERROR("Z_DATA_ERROR : The compressed data (referenced by source) was corrupted.");
+		}
+	}
+	return data;
+}
+} /* namespace find_object */

src/Compression.h

@@ -0,0 +1,20 @@
+/*
+ * Compression.h
+ *
+ *  Created on: Sep 10, 2018
+ *      Author: labm2414
+ */
+
+#ifndef SRC_COMPRESSION_H_
+#define SRC_COMPRESSION_H_
+
+#include <opencv2/opencv.hpp>
+
+namespace find_object {
+
+std::vector<unsigned char> compressData(const cv::Mat & data);
+cv::Mat uncompressData(const unsigned char * bytes, unsigned long size);
+
+}
+
+#endif /* SRC_COMPRESSION_H_ */

src/FindObject.cpp

@@ -735,7 +735,7 @@ protected:
 	{
 		QTime time;
 		time.start();
-		UINFO("Extracting descriptors from object %d...", objectId_);
+		UDEBUG("Extracting descriptors from object %d...", objectId_);
 
 		QTime timeStep;
 		timeStep.start();
@@ -850,6 +850,7 @@ private:
 
 void FindObject::updateObjects(const QList<int> & ids)
 {
+	UINFO("Update %d objects...", ids.size());
 	QList<ObjSignature*> objectsList;
 	if(ids.size())
 	{
@@ -884,7 +885,7 @@ void FindObject::updateObjects(const QList<int> & ids)
 
 		if(objectsList.size())
 		{
-			UINFO("Features extraction from %d objects...", objectsList.size());
+			UINFO("Features extraction from %d objects... (threads=%d)", objectsList.size(), threadCounts);
 			for(int i=0; i<objectsList.size(); i+=threadCounts)
 			{
 				QVector<ExtractFeaturesThread*> threads;
@@ -995,6 +996,8 @@ void FindObject::updateVocabulary(const QList<int> & ids)
 		}
 	}
 
+	UINFO("Updating vocabulary with %d objects and %d descriptors...", ids.size(), count);
+
 	// Copy data
 	if(count)
 	{
@@ -1100,6 +1103,10 @@ void FindObject::updateVocabulary(const QList<int> & ids)
 			}
 			if(addedWords && !Settings::getGeneral_vocabularyFixed())
 			{
+				if(!incremental)
+				{
+					UINFO("Updating vocabulary...");
+				}
 				vocabulary_->update();
 			}
 

src/MainWindow.cpp

@@ -244,15 +244,23 @@ MainWindow::MainWindow(FindObject * findObject, Camera * camera, QWidget * paren
 
 	if(findObject_->objects().size())
 	{
+		UINFO("Creating %d object widgets...", findObject_->objects().size());
 		// show objects already loaded in FindObject
+		int i=0;
 		for(QMap<int, ObjSignature *>::const_iterator iter = findObject_->objects().constBegin();
 			iter!=findObject_->objects().constEnd();
 			++iter)
 		{
-			ObjWidget * obj = new ObjWidget(iter.key(), iter.value()->keypoints(), iter.value()->words(), cvtCvMat2QImage(iter.value()->image()));
+			ObjWidget * obj = new ObjWidget(iter.key(), iter.value()->keypoints(), iter.value()->words(), iter.value()->image().empty()?QImage():cvtCvMat2QImage(iter.value()->image()));
 			objWidgets_.insert(obj->id(), obj);
 			this->showObject(obj);
+			++i;
+			if(i % 100 == 0)
+			{
+				UINFO("Created %d/%d widgets...", i, findObject_->objects().size());
+			}
 		}
+		UINFO("Creating %d object widgets... done!", findObject_->objects().size());
 		ui_->actionSave_objects->setEnabled(true);
 		ui_->actionSave_session->setEnabled(true);
 	}
@@ -1045,9 +1053,12 @@ void MainWindow::showObject(ObjWidget * obj)
 		ui_->verticalLayout_objects->insertLayout(ui_->verticalLayout_objects->count()-1, vLayout);
 
 		QByteArray ba;
-		QBuffer buffer(&ba);
+		if(obj->pixmap().width() > 0)
-		buffer.open(QIODevice::WriteOnly);
+		{
-		obj->pixmap().scaledToWidth(128).save(&buffer, "JPEG"); // writes image into JPEG format
+			QBuffer buffer(&ba);
+			buffer.open(QIODevice::WriteOnly);
+			obj->pixmap().scaledToWidth(128).save(&buffer, "JPEG"); // writes image into JPEG format
+		}
 		imagesMap_.insert(obj->id(), ba);
 
 		// update objects size slider
@@ -1411,6 +1422,14 @@ void MainWindow::update(const cv::Mat & image)
 			// add rectangle
 			QPen rectPen(obj->color());
 			rectPen.setWidth(Settings::getHomography_rectBorderWidth());
+			if(rect.isNull())
+			{
+				QMap<int, ObjSignature*>::const_iterator iter = findObject_->objects().constFind(id);
+				if(iter!=findObject_->objects().end())
+				{
+					rect = iter.value()->rect();
+				}
+			}
 			RectItem * rectItemScene = new RectItem(id, rect);
 			connect(rectItemScene, SIGNAL(hovered(int)), this, SLOT(rectHovered(int)));
 			rectItemScene->setPen(rectPen);

src/ObjSignature.h

@@ -34,6 +34,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <QtCore/QRect>
 #include <QtCore/QDataStream>
 #include <QtCore/QByteArray>
+#include <QtCore/QFileInfo>
+#include <Compression.h>
 
 namespace find_object {
 
@@ -84,18 +86,46 @@ public:
 								keypoints_.at(j).size;
 		}
 
-		qint64 dataSize = descriptors_.elemSize()*descriptors_.cols*descriptors_.rows;
+		std::vector<unsigned char> bytes  = compressData(descriptors_);
-		streamPtr << descriptors_.rows <<
+
-						descriptors_.cols <<
+		qint64 dataSize = bytes.size();
-						descriptors_.type() <<
+		int old = 0;
-						dataSize;
+		if(dataSize <= std::numeric_limits<int>::max())
-		streamPtr << QByteArray((char*)descriptors_.data, dataSize);
+		{
+			// old: rows, cols, type
+			streamPtr << old << old << old << dataSize;
+			streamPtr << QByteArray::fromRawData((const char*)bytes.data(), dataSize);
+		}
+		else
+		{
+			UERROR("Descriptors (compressed) are too large (%d MB) to be saved! Limit is 2 GB (based on max QByteArray size).",
+					dataSize/(1024*1024));
+			// old: rows, cols, type, dataSize
+			streamPtr << old << old << old << old;
+			streamPtr << QByteArray(); // empty
+		}
 
 		streamPtr << words_;
 
-		std::vector<unsigned char> bytes;
+		if(!image_.empty())
-		cv::imencode(".png", image_, bytes);
+		{
-		streamPtr << QByteArray((char*)bytes.data(), (int)bytes.size());
+			std::vector<unsigned char> bytes;
+			QString ext = QFileInfo(filePath_).suffix();
+			if(ext.isEmpty())
+			{
+				// default png
+				cv::imencode(".png", image_, bytes);
+			}
+			else
+			{
+				cv::imencode(std::string(".")+ext.toStdString(), image_, bytes);
+			}
+			streamPtr << QByteArray::fromRawData((const char*)bytes.data(), (int)bytes.size());
+		}
+		else
+		{
+			streamPtr << QByteArray();
+		}
 
 		streamPtr << rect_;
 	}
@@ -120,15 +150,34 @@ public:
 		int rows,cols,type;
 		qint64 dataSize;
 		streamPtr >> rows >> cols >> type >> dataSize;
-		QByteArray data;
+		if(rows == 0 && cols == 0 && type == 0)
-		streamPtr >> data;
+		{
-		descriptors_ = cv::Mat(rows, cols, type, data.data()).clone();
+			// compressed descriptors
+			UASSERT(dataSize <= std::numeric_limits<int>::max());
+			QByteArray data;
+			streamPtr >> data;
+			descriptors_ = uncompressData((unsigned const char*)data.data(), dataSize);
+		}
+		else
+		{
+			// old raw format
+			QByteArray data;
+			streamPtr >> data;
+			if(data.size())
+			{
+				descriptors_ = cv::Mat(rows, cols, type, data.data()).clone();
+			}
+			else if(dataSize)
+			{
+				UERROR("Error reading descriptor data for object=%d", id_);
+			}
+		}
 
 		streamPtr >> words_;
 
 		QByteArray image;
 		streamPtr >> image;
-		if(!ignoreImage)
+		if(!ignoreImage && image.size())
 		{
 			std::vector<unsigned char> bytes(image.size());
 			memcpy(bytes.data(), image.data(), image.size());

src/ObjWidget.cpp

@@ -266,8 +266,13 @@ void ObjWidget::setTextLabel(const QString & text)
 
 void ObjWidget::updateImage(const QImage & image)
 {
-	pixmap_ = QPixmap::fromImage(image);
+	pixmap_ = QPixmap();
-	rect_ = pixmap_.rect();
+	rect_ = QRect();
+	if(!image.isNull())
+	{
+		pixmap_ = QPixmap::fromImage(image);
+		rect_ = pixmap_.rect();
+	}
 	label_->setVisible(image.isNull());
 }
 void ObjWidget::updateData(const std::vector<cv::KeyPoint> & keypoints, const QMultiMap<int, int> & words)

src/Vocabulary.cpp

@@ -28,9 +28,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "find_object/Settings.h"
 
 #include "find_object/utilite/ULogger.h"
+#include "Compression.h"
 #include "Vocabulary.h"
 #include <QtCore/QVector>
 #include <QDataStream>
+#include <QTime>
 #include <stdio.h>
 #if CV_MAJOR_VERSION < 3
 #include <opencv2/gpu/gpu.hpp>
@@ -80,16 +82,31 @@ void Vocabulary::save(QDataStream & streamSessionPtr, bool saveVocabularyOnly) c
 	}
 	else
 	{
+		UINFO("Saving %d object references...", wordToObjects_.size());
 		streamSessionPtr << wordToObjects_;
 	}
 
 	// save words
-	qint64 dataSize = indexedDescriptors_.elemSize()*indexedDescriptors_.cols*indexedDescriptors_.rows;
+	qint64 rawDataSize = indexedDescriptors_.rows * indexedDescriptors_.cols * indexedDescriptors_.elemSize();
-	streamSessionPtr << indexedDescriptors_.rows <<
+	UINFO("Compressing words... (%dx%d, %d MB)", indexedDescriptors_.rows, indexedDescriptors_.cols, rawDataSize/(1024*1024));
-			indexedDescriptors_.cols <<
+	std::vector<unsigned char> bytes  = compressData(indexedDescriptors_);
-			indexedDescriptors_.type() <<
+	qint64 dataSize = bytes.size();
-			dataSize;
+	UINFO("Compressed = %d MB", dataSize/(1024*1024));
-	streamSessionPtr << QByteArray((char*)indexedDescriptors_.data, dataSize);
+	int old = 0;
+	if(dataSize <= std::numeric_limits<int>::max())
+	{
+		// old: rows, cols, type
+		streamSessionPtr << old << old << old << dataSize;
+		streamSessionPtr << QByteArray::fromRawData((const char*)bytes.data(), dataSize);
+	}
+	else
+	{
+		UERROR("Vocabulary (compressed) is too large (%d MB) to be saved! Limit is 2 GB (based on max QByteArray size).",
+				dataSize/(1024*1024));
+		// old: rows, cols, type, dataSize
+		streamSessionPtr << old << old << old << old;
+		streamSessionPtr << QByteArray(); // empty
+	}
 }
 
 void Vocabulary::load(QDataStream & streamSessionPtr, bool loadVocabularyOnly)
@@ -104,17 +121,45 @@ void Vocabulary::load(QDataStream & streamSessionPtr, bool loadVocabularyOnly)
 	}
 	else
 	{
+		UINFO("Loading words to objects references...");
 		streamSessionPtr >> wordToObjects_;
+		UINFO("Loaded %d object references...", wordToObjects_.size());
 	}
 
 	// load words
 	int rows,cols,type;
 	qint64 dataSize;
 	streamSessionPtr >> rows >> cols >> type >> dataSize;
-	QByteArray data;
+	if(rows == 0 && cols == 0 && type == 0)
-	streamSessionPtr >> data;
+	{
-	indexedDescriptors_ = cv::Mat(rows, cols, type, data.data()).clone();
+		// compressed vocabulary
+		UINFO("Loading words... (compressed format: %d MB)", dataSize/(1024*1024));
+		UASSERT(dataSize <= std::numeric_limits<int>::max());
+		QByteArray data;
+		streamSessionPtr >> data;
+		UINFO("Uncompress vocabulary...");
+		indexedDescriptors_ = uncompressData((unsigned const char*)data.data(), dataSize);
+		UINFO("Words: %dx%d (%d MB)", indexedDescriptors_.rows, indexedDescriptors_.cols,
+				(indexedDescriptors_.rows * indexedDescriptors_.cols * indexedDescriptors_.elemSize()) / (1024*1024));
+	}
+	else
+	{
+		// old raw format
+		UINFO("Loading words... (old format: %dx%d (%d MB))", rows, cols, dataSize/(1024*1024));
+		QByteArray data;
+		streamSessionPtr >> data;
+		UINFO("Allocate memory...");
+		if(data.size())
+		{
+			indexedDescriptors_ = cv::Mat(rows, cols, type, data.data()).clone();
+		}
+		else if(dataSize)
+		{
+			UERROR("Error reading vocabulary data...");
+		}
+	}
 
+	UINFO("Update vocabulary index...");
 	update();
 }