Server: use libjpeg-turbo when possible instead of relying on dlib's jpeg support

fuzun · fuzun · commit b7c4c1fd76d0 · 2021-03-28T23:16:36.000+03:00
diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt
@@ -17,6 +17,8 @@ set(CMAKE_AUTORCC ON)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
+SET(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}")
+
 if (MSVC)
     add_definitions(/bigobj)
     add_definitions(/FS)
@@ -67,6 +69,22 @@ FetchContent_Declare(dlib
 )
 FetchContent_MakeAvailable(dlib)
 
+if (NOT DLIB_JPEG_SUPPORT)
+    message(FATAL_ERROR "dlib's jpeg support must be enabled!")
+endif()
+
+find_package(TurboJPEG)
+if (TurboJPEG_FOUND OR DLIB_JPEG_SUPPORT)
+    target_include_directories(iot-facerecognition-server PRIVATE ${TurboJPEG_INCLUDE_DIRS})
+    target_link_libraries(iot-facerecognition-server PRIVATE ${TurboJPEG_LIBRARIES})
+    target_compile_definitions(iot-facerecognition-server PRIVATE TURBOJPEG_AVAILABLE)
+
+    message(STATUS "libjpeg-turbo is found. It will be used for fast JPEG decoding.")
+elseif (DLIB_JPEG_SUPPORT)
+    message(WARNING "libjpeg-turbo is NOT found. Real-time JPEG decoding may be slow.")
+endif()
+
+
 target_include_directories(iot-facerecognition-server
 	PRIVATE
 		source/ClientDialog
diff --git a/server/source/Client/Client.cpp b/server/source/Client/Client.cpp
@@ -125,7 +125,10 @@ void Client::processBinaryMessage(const QByteArray& data)
 {
     if(dialog)
     {
-        dlib::array2d<dlib::rgb_pixel> img(DLIBWorker::constructImgFromBuffer(data));
+        dlib::array2d<dlib::rgb_pixel> img(DLIBWorker::decodeJPEG(data, nullptr));
+
+        if (img.size() <= 0)
+            return;
 
         QPixmap pixmap(QPixmap::fromImage(QImage((unsigned char*)dlib::image_data(img), img.nc(), img.nr(), QImage::Format_RGB888)));
         primaryDisplay->setPixmap(pixmap);
diff --git a/server/source/DLIBWorker/DLIBWorker.cpp b/server/source/DLIBWorker/DLIBWorker.cpp
@@ -30,13 +30,13 @@
 
 #include <Client/Client.h>
 
-#ifndef DLIB_JPEG_SUPPORT
-#error "DLIB must have built-in libjpeg support enabled!"
-#endif
-
 #include <dlib/clustering.h>
 #include <dlib/image_io.h>
 
+#ifdef TURBOJPEG_AVAILABLE
+#include <turbojpeg.h>
+#endif
+
 #include "config.h"
 
 using namespace dlib;
@@ -94,6 +94,104 @@ DLIBWorker::DLIBWorker(class QSettings* config, const Settings *settings)
             }
         }
     }
+
+#ifdef TURBOJPEG_AVAILABLE
+    QMetaObject::invokeMethod(this, [this]() {
+            m_tjHandle = tjInitDecompress();
+        }, Qt::QueuedConnection);
+#endif
+}
+
+DLIBWorker::~DLIBWorker()
+{
+#ifdef TURBOJPEG_AVAILABLE
+    if (m_tjHandle)
+        tjDestroy(m_tjHandle);
+#endif
+}
+
+dlib::array2d<rgb_pixel> DLIBWorker::decodeJPEG(const QByteArray &jpegBuffer, void *tjHandle)
+{
+    array2d<rgb_pixel> img;
+
+#ifdef TURBOJPEG_AVAILABLE
+    bool destroyHandle = false;
+
+    if (tjHandle == nullptr)
+    {
+        tjHandle = tjInitDecompress();
+        destroyHandle = true;
+    }
+
+    assert(tjHandle);
+
+    int width, height;
+    int jpegSubsamp, jpegColorspace;
+
+    int ret;
+
+    auto ptrBuffer = reinterpret_cast<const unsigned char*>(jpegBuffer.constData());
+
+    ret = tjDecompressHeader3(tjHandle,
+        ptrBuffer,
+        jpegBuffer.size(),
+        &width, &height,
+        &jpegSubsamp, &jpegColorspace);
+
+    if (ret || width <= 0 || height <= 0)
+    {
+        if (destroyHandle)
+            tjDestroy(tjHandle);
+
+        return img;
+    }
+
+    img.set_size(height, width);
+
+    tjDecompress2(tjHandle,
+                  ptrBuffer,
+                  jpegBuffer.size(),
+                  static_cast<unsigned char*>(image_data(img)),
+                  width, 0, height,
+                  TJPF_RGB,
+                  TJFLAG_FASTDCT);
+
+    if (destroyHandle)
+        tjDestroy(tjHandle);
+#else
+    Q_UNUSED(tjHandle);
+
+    try
+    {
+        load_jpeg(img, buffer.constData(), buffer.size());
+    }
+    catch(...)
+    {
+
+    }
+#endif
+
+    return img;
+}
+
+dlib::array2d<rgb_pixel> DLIBWorker::decodeJPEG(const QByteArray &jpegBuffer)
+{
+    auto decoded = decodeJPEG(jpegBuffer,
+#ifdef TURBOJPEG_AVAILABLE
+        m_tjHandle
+#else
+        nullptr
+#endif
+        );
+
+    if (decoded.size() <= 0)
+#ifdef TURBOJPEG_AVAILABLE
+        emit log(QString("libjpeg-turbo error code: %1").arg(tjGetErrorStr2(m_tjHandle)));
+#else
+        emit log("dlib::load_jpeg failed!");
+#endif
+
+    return decoded;
 }
 
 std::vector<Face> DLIBWorker::findFaces(const array2d<rgb_pixel>& img)
@@ -260,26 +358,14 @@ void DLIBWorker::process(const QByteArray& buffer)
 
     try
     {
-        if(referenceFaces.size() == 0)
-        {
-            if(m_refPhotoFileList.size() == 0)
-            {
-                throwException(std::exception("Reference face list is empty!"));
-            }
-            else
-            {
-                detector = get_frontal_face_detector();
-                string landmarkModel = m_faceLandmarkModelFile.toStdString();
-                string recogModel = m_faceRecognitionModelFile.toStdString();
-                deserialize(landmarkModel) >> sp;
-                deserialize(recogModel) >> net;
+        auto img = decodeJPEG(buffer);
 
-                setupReference(m_refPhotoFileList);
-            }
+        if (img.size() <= 0)
+        {
+            m_busy = false;
+            return;
         }
 
-        auto img = constructImgFromBuffer(buffer);
-
         if (m_settings->objectDetectionEnabled)
         {
             if (labels.empty())
@@ -312,30 +398,51 @@ void DLIBWorker::process(const QByteArray& buffer)
 
         if (m_settings->faceRecognitionEnabled)
         {
+            if(referenceFaces.size() == 0)
+            {
+                if(m_refPhotoFileList.size() == 0)
+                {
+                    throwException(std::exception("Reference face list is empty!"));
+                }
+                else
+                {
+                    detector = get_frontal_face_detector();
+                    string landmarkModel = m_faceLandmarkModelFile.toStdString();
+                    string recogModel = m_faceRecognitionModelFile.toStdString();
+                    deserialize(landmarkModel) >> sp;
+                    deserialize(recogModel) >> net;
+
+                    setupReference(m_refPhotoFileList);
+                }
+            }
+
             auto faces = findFaces(img);
 
-            rectangle rect;
-            for (const auto& face : referenceFaces)
+            if (faces.size() > 0)
             {
-                faces.push_back(make_tuple(get<0>(face), rect, get<2>(face)));
-            }
+                rectangle rect;
+                for (const auto& face : referenceFaces)
+                {
+                    faces.push_back(make_tuple(get<0>(face), rect, get<2>(face)));
+                }
 
-            auto graph = createGraph(faces, m_threshold);
+                auto graph = createGraph(faces, m_threshold);
 
-            auto clusters = cluster(graph, faces);
+                auto clusters = cluster(graph, faces);
 
-            QVector<QPair<QRect, QString>> linearFaces;
+                QVector<QPair<QRect, QString>> linearFaces;
 
-            for (const auto& it : clusters)
-            {
-                QString str = it.first;
-                for (const auto& it2 : it.second)
+                for (const auto& it : clusters)
                 {
-                    linearFaces.push_back(qMakePair(QRect(it2.left(), it2.top(), it2.width(), it2.height()), str));
+                    QString str = it.first;
+                    for (const auto& it2 : it.second)
+                    {
+                        linearFaces.push_back(qMakePair(QRect(it2.left(), it2.top(), it2.width(), it2.height()), str));
+                    }
                 }
-            }
 
-            emit doneFace(linearFaces);
+                emit doneFace(linearFaces);
+            }
         }
 
         m_busy = false;
@@ -346,9 +453,3 @@ void DLIBWorker::process(const QByteArray& buffer)
     }
 }
 
-dlib::array2d<dlib::rgb_pixel> DLIBWorker::constructImgFromBuffer(const QByteArray& buffer)
-{
-    array2d<rgb_pixel> img;
-    load_jpeg(img, buffer.constData(), buffer.size());
-    return img;
-}
diff --git a/server/source/DLIBWorker/DLIBWorker.h b/server/source/DLIBWorker/DLIBWorker.h
@@ -108,6 +108,10 @@ class DLIBWorker : public QObject
     QString                          m_imageNetClassifierFile;
     size_t                           m_numCrops;
 
+#ifdef TURBOJPEG_AVAILABLE
+    void* m_tjHandle = nullptr;
+#endif
+
     const struct Settings* m_settings;
 
     std::vector<Face> referenceFaces;
@@ -123,8 +127,10 @@ public slots:
 
 public:
     explicit DLIBWorker(class QSettings* config, const struct Settings* settings);
+    ~DLIBWorker();
 
-    static dlib::array2d<dlib::rgb_pixel> constructImgFromBuffer(const QByteArray& buffer);
+    static dlib::array2d<dlib::rgb_pixel> decodeJPEG(const QByteArray& jpegBuffer, void* tjHandle);
+    dlib::array2d<dlib::rgb_pixel> decodeJPEG(const QByteArray& jpegBuffer);
 
     inline bool isBusy() const { return m_busy; };
 

Original file line number	Diff line number	Diff line change
`@@ -125,7 +125,10 @@ void Client::processBinaryMessage(const QByteArray& data)`
`125`	`125`	`{`
`126`	`126`	`if(dialog)`
`127`	`127`	`{`
`128`		`- dlib::array2d<dlib::rgb_pixel> img(DLIBWorker::constructImgFromBuffer(data));`
	`128`	`+ dlib::array2d<dlib::rgb_pixel> img(DLIBWorker::decodeJPEG(data, nullptr));`
	`129`	`+`
	`130`	`+ if (img.size() <= 0)`
	`131`	`+ return;`
`129`	`132`
`130`	`133`	`QPixmap pixmap(QPixmap::fromImage(QImage((unsigned char*)dlib::image_data(img), img.nc(), img.nr(), QImage::Format_RGB888)));`
`131`	`134`	`primaryDisplay->setPixmap(pixmap);`