Version 0.9.2: re-implemented SVM using Allen's suggested feature extraction method;

added options in demo to turn on/off SVM, edge connection test, area display;
disabled second pass of hand detection for efficiency concerns;
modified several default parameter values in ObjectParams;
fixed two minor bugs in hand detection
removed #include<stdafx.h> from header files and moved definitions to version.h to speed up external project compilation

Author: sxyu

CMakeLists.txt

@@ -4,7 +4,7 @@ project( OpenARK )
 
 set(OpenARK_VERSION_MAJOR 0)
 set(OpenARK_VERSION_MINOR 9)
-set(OpenARK_VERSION_PATCH 1)
+set(OpenARK_VERSION_PATCH 2)
 
 set(SVM_PATHS "\"svm/\", \"../svm/\", \"\"")
 

Converter.h

@@ -1,8 +1,9 @@
 #pragma once
 
-#include "stdafx.h"
 #include "version.h"
 
+#include <opencv2/core.hpp>
+
 //Intel RealSense 3D SDK libraries
 #include "RealSense/SampleReader.h"
 #include "RealSense/Session.h"

DepthCamera.cpp

@@ -89,11 +89,15 @@ namespace ark {
         cv::Mat pendingClusters;
         if (elim_planes) pendingClusters = cv::Mat::zeros(R, C, CV_8U);
 
+        // number of 'pending' clusters
+        uchar pendingClusterCount = 0;
+
         float bestHandDist = FLT_MAX;
 
         // 2. eliminate large planes
 
-        const int DOMINANT_PLANE_MIN_POINTS = params->dominantPlaneMinPoints * R * C /
+        // compute exact constant
+        static const int DOMINANT_PLANE_MIN_POINTS = params->dominantPlaneMinPoints * R * C /
                                         (params->normalResolution * params->normalResolution);
 
         if (elim_planes){
@@ -107,41 +111,14 @@ namespace ark {
             }
         }
 
-        // 3. first pass: try to detect hands directly
-        // number of 'pending' clusters
-        uchar pendingClusterCount = 0;
+        // 3. detect hands directly
         boost::shared_ptr<Hand> bestHandObject =
-            detectHandHelper(xyzMap, hands, params, &bestHandDist, &pendingClusters, 
-                &pendingClusterCount);
-
-        // 4. second pass: eliminate planes and try again
-        if (elim_planes) {
-            cv::Vec3b color;
-
-            for (uchar k = 0; k < pendingClusterCount; ++k) {
-                color = util::paletteColor(k);
-
-                uchar maskColor = 255 - k;
-
-                std::vector<Vec3f> equations;
-                std::vector<VecP2iPtr> points;
-                std::vector<VecV3fPtr> pointsXYZ;
-
-                detectPlaneHelper(xyzMap, equations, points, pointsXYZ, params, nullptr,
-                    &pendingClusters, maskColor);
-
-                if (equations.size()) {
-                    for (uint i = 0; i < equations.size(); ++i) {
-                        util::removePlane(xyzMap, equations[i], params->handPlaneMinNorm);
-                    }
-
-                    bestHandObject = detectHandHelper(xyzMap, hands, params, 
-                        &bestHandDist, nullptr, nullptr, &pendingClusters, maskColor);
-                }
-            }
-        }
+            detectHandHelper(xyzMap, hands, params, &bestHandDist, 
+                elim_planes ? &pendingClusters : nullptr,
+                elim_planes ? &pendingClusterCount : nullptr);
 
         if (bestHandObject != nullptr) {
+            // if no hands surpass 'high confidence threshold', at least add one hand
             hands.push_back(bestHandObject);
         }
 
@@ -629,8 +606,8 @@ namespace ark {
         if (best_hand_dist) closestHandDist = *best_hand_dist;
 
         // 2. try to directly cluster objects using flood fill
-        std::vector<Point2i> allIjPoints(R * C + 1);
-        std::vector<Vec3f> allXyzPoints(R * C + 1);
+        std::vector<Point2i> allIjPoints(R * C);
+        std::vector<Vec3f> allXyzPoints(R * C);
 
         // compute the minimum number of points in a cluster according to params
         const int CLUSTER_MIN_POINTS = (int) (params->handClusterMinPoints * R * C);

DepthCamera.h

@@ -388,7 +388,7 @@ namespace ark {
         cv::Mat ampMapBuf;
         cv::Mat flagMapBuf;
 
-        /** Mutexes to ensure thread safety while updating images 
+        /** Mutex to ensure thread safety while updating images 
          *  (mutable = modificable even to const methods)
          */
         mutable std::mutex imageMutex;

FrameObject.h

@@ -1,8 +1,10 @@
 #pragma once
 
-#include "stdafx.h" 
 #include "version.h"
 
+#include <opencv2/core.hpp>
+#include <vector>
+
 // OpenARK headers
 #include "version.h"
 #include "ObjectParams.h"

Hand.cpp

@@ -1,9 +1,9 @@
-#include "stdafx.h"
 #include "version.h"
 
 #include "Util.h"
 #include "Hand.h"
 #include "Visualizer.h"
+#include "HandClassifier.h"
 
 // limited to file scope
 namespace {
@@ -54,8 +54,8 @@ namespace {
 
 namespace ark {
 
-    // Initialize the SVM hand classifier
-    classifier::HandClassifier & Hand::handClassifier = classifier::SVMHandClassifier(SVM_PATHS);
+    // Initialize the SVM hand validator
+    static classifier::SVMHandValidator & handValidator = classifier::SVMHandValidator(SVM_PATHS);
 
     Hand::Hand() : FrameObject() { }
 
@@ -93,30 +93,18 @@ namespace ark {
         }  
     }
 
-    bool Hand::checkForHand() 
+    bool Hand::checkForHand()
     {
 #ifdef DEBUG
         cv::Mat visual = cv::Mat::zeros(fullMapSize.height, fullMapSize.width, CV_8UC3);
+        cv::Mat defectVisual = cv::Mat::zeros(fullMapSize.height, fullMapSize.width, CV_8UC3);
 #endif
         if (points->size() == 0 || num_points == 0) {
             return false;
         }
 
         computeContour(xyzMap, points.get(), points_xyz.get(), topLeftPt, num_points);
 
-        // recompute convex hull based on new contour
-        convexHull.clear(); getConvexHull();
-
-        // Begin by computing defects
-        std::vector<cv::Vec4i> defects;
-
-        if (indexHull.size() > 3)
-        {
-            std::vector<int> tmpHull;
-
-            cv::convexityDefects(contour, indexHull, defects);
-        }
-
         // ** Find center of contour **
         Point2i centroid = findCenter(contour) - topLeftPt;
 
@@ -125,7 +113,7 @@ namespace ark {
 
         // Find radius and center point of largest inscribed circle above center
         Vec3f topPt = util::averageAroundPoint(xyzMap, (*points)[0] - topLeftPt,
-                                                 params->xyzAverageSize);
+            params->xyzAverageSize);
 
         // radius of largest inscribed circle
         double cirrad;
@@ -243,6 +231,10 @@ namespace ark {
             return false;
         }
 
+        if (contour[wristL].x > contour[wristR].x) {
+            std::swap(wristL, wristR);
+        }
+
         wristL_ij = contour[wristL];
         wristR_ij = contour[wristR];
         wristL_xyz = util::averageAroundPoint(xyzMap,
@@ -268,7 +260,54 @@ namespace ark {
             return false;
         }
 
+        // (finished detecting wrist)
+
+        // ** Remove everything below wrist **
+
+        std::vector<Point2i> aboveWristPointsIJ;
+        std::vector<Vec3f> aboveWristPointsXYZ;
+
+        if (wristR_ij.x != wristL_ij.x) {
+            double slope = (double)(wristR_ij.y - wristL_ij.y) / (wristR_ij.x - wristL_ij.x);
+
+            for (int i = 0; i < num_points; ++i) {
+                const Point2i & pt = (*points)[i];
+                double y_hat = wristL_ij.y + (pt.x - wristL_ij.x) * slope;
+
+                Vec3f & vec = xyzMap.at<Vec3f>(pt - topLeftPt);
+
+                if (pt.y > y_hat) {
+                   vec = 0;
+                }
+                else {
+                    aboveWristPointsIJ.push_back(pt);
+                    aboveWristPointsXYZ.push_back(vec);
+                }
+            }
+        }
+
+        num_points = (int)aboveWristPointsIJ.size();
+        points->swap(aboveWristPointsIJ);
+        points_xyz->swap(aboveWristPointsXYZ);
+
+        // recompute contour
+        computeContour(xyzMap, points.get(), points_xyz.get(), topLeftPt, num_points);
+
         // ** Detect fingers **
+
+        // recompute convex hull based on new contour
+        convexHull.clear(); getConvexHull();
+
+        // compute defects
+        std::vector<cv::Vec4i> defects;
+
+        if (indexHull.size() > 3)
+        {
+            std::vector<int> tmpHull;
+
+            cv::convexityDefects(contour, indexHull, defects);
+        }
+
         // sort all defects found by angle
         DefectComparer comparer(contour, defects, this->palmCenterIJ);
         std::sort(defects.begin(), defects.end(), comparer);
@@ -290,19 +329,11 @@ namespace ark {
             // contains info about the defect
             cv::Vec4i defect = defects[i];
 
-            // skip if defect is under wrist
-            if (direction == -1) {
-                if (wristL <= wristR) {
-                    if (defect[2] >= wristL && defect[2] <= wristR) continue;
-                }
-                else if (defect[2] <= wristR || defect[2] >= wristL) continue;
-            }
-            else {
-                if (wristL <= wristR) {
-                    if (defect[2] <= wristL || defect[2] >= wristR) continue;
-                }
-                else if (defect[2] >= wristR && defect[2] <= wristL) continue;
-            }
+#ifdef DEBUG
+            cv::line(defectVisual, contour[defects[i][0]], contour[defects[i][2]], cv::Scalar(0, 255, 0));
+            cv::line(defectVisual, contour[defects[i][1]], contour[defects[i][2]], cv::Scalar(0, 0, 255));
+            cv::circle(defectVisual, contour[defects[i][2]], 5, cv::Scalar(255,255,255), 2);
+#endif
 
             // point on convex hull where defect begins. fingertip candidate
             Point2i start = contour[defect[0]] - topLeftPt;
@@ -397,6 +428,8 @@ namespace ark {
             cv::Scalar(0, 255, 255), 2);
         cv::rectangle(visual, wristL_ij - Point2i(10, 10), wristL_ij + Point2i(10, 10),
             cv::Scalar(0, 255, 255), 2);
+
+        cv::imshow("[Hand Defects Debug]", defectVisual);
 #endif
 
         // select fingers from candidates
@@ -684,20 +717,28 @@ namespace ark {
         cv::imshow("[Hand Debug]", visual);
 #endif
 
+        size_t nFin = this->fingersIJ.size();
+
         // report not hand if there are too few/many fingers
-        if (this->fingersIJ.size() > 6 || this->fingersIJ.size() < 1) {
+        if (nFin > 6 || nFin < 1) {
             return false;
         }
+        
+        // find dominant direction
+
+        if (nFin > 0) {
+            Point2f fingCen = this->fingersIJ[nFin / 2];
+            if (nFin % 2 == 0) {
+                fingCen += Point2f(this->fingersIJ[nFin / 2 - 1]);
+                fingCen /= 2.0f;
+            }
 
-        // Final SVM check
-        if (params->handUseSVM && handClassifier.isTrained()) {
-            this->isHand = true;
-
-            std::vector<double> features =
-                classifier::HandClassifier::extractHandFeatures(*this, xyzMap, topLeftPt, 1.0,
-                                                          fullMapSize.width);
+            this->dominantDir = fingCen - Point2f(this->palmCenterIJ);
+        }
 
-            this->svmConfidence = handClassifier.classify(features);
+        // Final SVM check
+        if (params->handUseSVM && handValidator.isTrained()) {
+            this->svmConfidence = handValidator.classify(*this, xyzMap, 32, 5, topLeftPt, fullMapSize.width);
             if (this->svmConfidence < params->handSVMConfidenceThresh) {
                 // SVM confidence value below threshold, reverse decision & destroy the hand instance
                 return false;
@@ -868,7 +909,12 @@ namespace ark {
         return circleRadius;
     }
 
-    double Hand::getSVMConfidence() const 
+    Point2f Hand::getDominantDirection() const
+    {
+        return dominantDir;
+    }
+
+    float Hand::getSVMConfidence() const
     {
         return svmConfidence;
     }

Hand.h

@@ -1,8 +1,7 @@
 #pragma once
-#include "stdafx.h"
+
 #include "FrameObject.h"
 #include "FramePlane.h"
-#include "HandClassifier.h"
 #include "version.h"
 
 namespace ark {
@@ -112,10 +111,16 @@ namespace ark {
         */
         double getCircleRadius() const;
 
+        /**
+        * Get a unit vector pointing towards the "dominant" direction of the hand
+        * i.e. from the center towards the fingers
+        */
+        Point2f getDominantDirection() const;
+
         /**
         * Get the SVM confidence rating of this object, [0.0, 1.0] (higher = more likely to be hand)
         */
-        double getSVMConfidence() const;
+        float getSVMConfidence() const;
 
         /**
         * True if hand cluster touches the bottom edge
@@ -172,11 +177,6 @@ namespace ark {
         */
         bool isValidHand() const;
 
-        /**
-        * SVM Hand classifier instance
-        */
-        static classifier::HandClassifier & handClassifier;
-
     protected:
         /**
          * Amount the gray map is scaled by for contour finding.
@@ -243,11 +243,16 @@ namespace ark {
          */
         double circleRadius;
 
+        /**
+         * stores the dominant direction of hand
+         */
+        Point2f dominantDir;
+
         /**
         * The confidence value (in [0, 1]) assigned to this hand by the SVM classifier,
         * higher = more likely to be hand
         */
-        double svmConfidence;
+        float svmConfidence;
 
         /**
         * Whether the hand object is actually valid
@@ -266,7 +271,6 @@ namespace ark {
         */
         bool rightEdgeConnected = false;
 
-
     };
 
     /* Shared pointer for Hand **/