Merge pull request #72 from HyperInspire/dev/eye_status

Dev/eye status

Former-commit-id: 610f84b

Author: tunmx

CMakeLists.txt

@@ -9,7 +9,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
 # Current version
 set(INSPIRE_FACE_VERSION_MAJOR 1)
 set(INSPIRE_FACE_VERSION_MINOR 1)
-set(INSPIRE_FACE_VERSION_PATCH 2)
+set(INSPIRE_FACE_VERSION_PATCH 3)
 
 # Converts the version number to a string
 string(CONCAT INSPIRE_FACE_VERSION_MAJOR_STR ${INSPIRE_FACE_VERSION_MAJOR})

cpp/inspireface/CMakeLists.txt

@@ -55,6 +55,7 @@ set(SOURCE_FILES ${SOURCE_FILES} ${CMAKE_CURRENT_SOURCE_DIR}/middleware/model_ar
 link_directories(${MNN_LIBS})
 
 if(ISF_BUILD_SHARED_LIBS)
+    add_definitions("-DISF_BUILD_SHARED_LIBS")
     add_library(InspireFace SHARED ${SOURCE_FILES})
 else()
     add_library(InspireFace STATIC ${SOURCE_FILES})

cpp/inspireface/c_api/inspireface.cc

@@ -281,6 +281,33 @@ HResult HFGetFaceBasicTokenSize(HPInt32 bufferSize) {
     return HSUCCEED;
 }
 
+HResult HFGetNumOfFaceDenseLandmark(HPInt32 num) {
+    *num = 106;
+    return HSUCCEED;
+}
+
+HResult HFGetFaceDenseLandmarkFromFaceToken(HFFaceBasicToken singleFace, HPoint2f* landmarks, HInt32 num) {
+    if (num != 106) {
+        return HERR_SESS_LANDMARK_NUM_NOT_MATCH;
+    }
+    inspire::FaceBasicData data;
+    data.dataSize = singleFace.size;
+    data.data = singleFace.data;
+    HyperFaceData face = {0};
+    HInt32 ret;
+    ret = DeserializeHyperFaceData((char* )data.data, data.dataSize, face);
+    if (ret != HSUCCEED) {
+        return ret;
+    }
+    for (size_t i = 0; i < num; i++)
+    {
+        landmarks[i].x = face.densityLandmark[i].x;
+        landmarks[i].y = face.densityLandmark[i].y;
+    }
+    
+    return HSUCCEED;
+}
+
 HResult HFFeatureHubFaceSearchThresholdSetting(float threshold) {
     FEATURE_HUB->SetRecognitionThreshold(threshold);
     return HSUCCEED;
@@ -549,7 +576,7 @@ HResult HFMultipleFacePipelineProcessOptional(HFSession session, HFImageStream s
     }
     if (customOption & HF_ENABLE_INTERACTION) {
         param.enable_interaction_liveness = true;
-    }
+    } 
 
 
     HResult ret;
@@ -633,6 +660,21 @@ HResult HFFaceQualityDetect(HFSession session, HFFaceBasicToken singleFace, HFlo
 
 }
 
+HResult HFGetFaceIntereactionResult(HFSession session, PHFFaceIntereactionResult result) {
+     if (session == nullptr) {
+        return HERR_INVALID_CONTEXT_HANDLE;
+    }
+    HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession* ) session;
+    if (ctx == nullptr) {
+        return HERR_INVALID_CONTEXT_HANDLE;
+    }
+    result->num = ctx->impl.GetFaceInteractionLeftEyeStatusCache().size();
+    result->leftEyeStatusConfidence = (HFloat* )ctx->impl.GetFaceInteractionLeftEyeStatusCache().data();
+    result->rightEyeStatusConfidence = (HFloat* )ctx->impl.GetFaceInteractionRightEyeStatusCache().data();
+
+    return HSUCCEED;
+}
+
 HResult HFFeatureHubGetFaceCount(HInt32* count) {
     *count = FEATURE_HUB->GetFaceFeatureCount();
     return HSUCCEED;

cpp/inspireface/c_api/inspireface.h

@@ -10,7 +10,7 @@
 #include "herror.h"
 
 #if defined(_WIN32)
-#ifdef HYPER_BUILD_SHARED_LIB
+#ifdef ISF_BUILD_SHARED_LIBS
 #define HYPER_CAPI_EXPORT __declspec(dllexport)
 #else
 #define HYPER_CAPI_EXPORT
@@ -298,6 +298,23 @@ HYPER_CAPI_EXPORT extern HResult HFCopyFaceBasicToken(HFFaceBasicToken token, HP
  */
 HYPER_CAPI_EXPORT extern HResult HFGetFaceBasicTokenSize(HPInt32 bufferSize);
 
+/**
+ * @brief Retrieve the number of dense facial landmarks.
+ * @param num Number of dense facial landmarks
+ * @return HResult indicating the success or failure of the operation.
+ */
+HYPER_CAPI_EXPORT extern HResult HFGetNumOfFaceDenseLandmark(HPInt32 num);
+
+/**
+ * @brief When you pass in a valid facial token, you can retrieve a set of dense facial landmarks. 
+ *          The memory for the dense landmarks must be allocated by you.
+ * @param singleFace Basic token representing a single face.
+ * @param landmarks Pre-allocated memory address of the array for 2D floating-point coordinates.
+ * @param num Number of landmark points
+ * @return HResult indicating the success or failure of the operation.
+ */
+HYPER_CAPI_EXPORT extern HResult HFGetFaceDenseLandmarkFromFaceToken(HFFaceBasicToken singleFace, HPoint2f* landmarks, HInt32 num);
+
 /************************************************************************
 * Face Recognition
 ************************************************************************/
@@ -618,6 +635,18 @@ HYPER_CAPI_EXPORT extern HResult HFGetFaceQualityConfidence(HFSession session, P
  */
 HYPER_CAPI_EXPORT extern HResult HFFaceQualityDetect(HFSession session, HFFaceBasicToken singleFace, HFloat *confidence);
 
+
+/**
+ * @brief Some facial states in the face interaction module.
+ */
+typedef struct HFFaceIntereactionResult {
+    HInt32 num;                             ///< Number of faces detected.
+    HPFloat leftEyeStatusConfidence;        ///< Left eye state: confidence close to 1 means open, close to 0 means closed.
+    HPFloat rightEyeStatusConfidence;       ///< Right eye state: confidence close to 1 means open, close to 0 means closed.
+} HFFaceIntereactionResult, *PHFFaceIntereactionResult;
+
+HYPER_CAPI_EXPORT extern HResult HFGetFaceIntereactionResult(HFSession session, PHFFaceIntereactionResult result);
+
 /************************************************************************
 * System Function
 ************************************************************************/

cpp/inspireface/c_api/intypedef.h

@@ -31,5 +31,9 @@ typedef struct HFaceRect {
     HInt32 height;        ///< Height of the rectangle.
 } HFaceRect;         ///< Rectangle representing a face region.
 
+typedef struct HPoint2f{
+    HFloat x;          ///< X-coordinate
+    HFloat y;          ///< Y-coordinate
+} HPoint2f;
 
 #endif //HYPERFACEREPO_INTYPEDEF_H

cpp/inspireface/common/face_data/data_tools.h

@@ -96,6 +96,15 @@ inline HyperFaceData INSPIRE_API FaceObjectToHyperFaceData(const FaceObject& obj
     data.face3DAngle.pitch = obj.high_result.pitch;
     data.face3DAngle.roll = obj.high_result.roll;
     data.face3DAngle.yaw = obj.high_result.yaw;
+    
+
+    const auto &lmk = obj.landmark_smooth_aux_.back();
+    for (size_t i = 0; i < lmk.size(); i++)
+    {
+        data.densityLandmark[i].x = lmk[i].x;
+        data.densityLandmark[i].y = lmk[i].y;
+    }
+    
 
     return data;
 }

cpp/inspireface/common/face_data/face_data_type.h

@@ -57,15 +57,16 @@ typedef struct TransMatrix {
 * Struct to represent hyper face data.
 */
 typedef struct HyperFaceData {
-    int trackState;         ///< Track state
-    int inGroupIndex;       ///< Index within a group
-    int trackId;            ///< Track ID
-    int trackCount;         ///< Track count
-    FaceRect rect;          ///< Face rectangle
-    TransMatrix trans;      ///< Transformation matrix
-    Point2F keyPoints[5];   ///< Key points (e.g., landmarks)
-    Face3DAngle face3DAngle; ///< 3D face angles
-    float quality[5];       ///< Quality values for key points
+    int trackState;                 ///< Track state
+    int inGroupIndex;               ///< Index within a group
+    int trackId;                    ///< Track ID
+    int trackCount;                 ///< Track count
+    FaceRect rect;                  ///< Face rectangle
+    TransMatrix trans;              ///< Transformation matrix
+    Point2F keyPoints[5];           ///< Key points (e.g., landmarks)
+    Face3DAngle face3DAngle;        ///< 3D face angles
+    float quality[5];               ///< Quality values for key points
+    Point2F densityLandmark[106];   ///< Face density landmark
 } HyperFaceData;
 
 } // namespace inspire

cpp/inspireface/common/face_info/face_object.h

@@ -312,6 +312,10 @@ class INSPIRE_API FaceObject {
         face_id_ = id;
     }
 
+    std::vector<float> left_eye_status_;
+    
+    std::vector<float> right_eye_status_;
+
 private:
     TRACK_STATE tracking_state_;
 //  std::shared_ptr<FaceAction> face_action_;

cpp/inspireface/face_context.cpp

@@ -62,6 +62,8 @@ int32_t FaceContext::FaceDetectAndTrack(CameraStream &image) {
     m_yaw_results_cache_.clear();
     m_pitch_results_cache_.clear();
     m_quality_score_results_cache_.clear();
+    m_react_left_eye_results_cache_.clear();
+    m_react_right_eye_results_cache_.clear();
     if (m_face_track_ == nullptr) {
         return HERR_SESS_TRACKER_FAILURE;
     }
@@ -129,6 +131,8 @@ int32_t FaceContext::FacesProcess(CameraStream &image, const std::vector<HyperFa
     std::lock_guard<std::mutex> lock(m_mtx_);
     m_mask_results_cache_.resize(faces.size(), -1.0f);
     m_rgb_liveness_results_cache_.resize(faces.size(), -1.0f);
+    m_react_left_eye_results_cache_.resize(faces.size(), -1.0f);
+    m_react_right_eye_results_cache_.resize(faces.size(), -1.0f);
     for (int i = 0; i < faces.size(); ++i) {
         const auto &face = faces[i];
         // RGB Liveness Detect
@@ -161,6 +165,38 @@ int32_t FaceContext::FacesProcess(CameraStream &image, const std::vector<HyperFa
                 return ret;
             }
         }
+        // Face interaction
+        if (param.enable_interaction_liveness) {
+            auto ret = m_face_pipeline_->Process(image, face, PROCESS_INTERACTION);
+            if (ret != HSUCCEED) {
+                return ret;
+            }
+            // Get eyes status
+            m_react_left_eye_results_cache_[i] = m_face_pipeline_->eyesStatusCache[0];
+            m_react_right_eye_results_cache_[i] = m_face_pipeline_->eyesStatusCache[1];
+            // Special handling:  ff it is a tracking state, it needs to be filtered
+            if (face.trackState > 0)
+            {   
+                auto idx = face.inGroupIndex;
+                if (idx < m_face_track_->trackingFace.size()) {
+                    auto& target = m_face_track_->trackingFace[idx];
+                    if (target.GetTrackingId() == face.trackId) {
+                        auto new_eye_left = EmaFilter(m_face_pipeline_->eyesStatusCache[0], target.left_eye_status_, 8, 0.2f);
+                        auto new_eye_right = EmaFilter(m_face_pipeline_->eyesStatusCache[1], target.right_eye_status_, 8, 0.2f);
+                        if (face.trackState > 1) {
+                            // The filtered value can be obtained only in the tracking state
+                            m_react_left_eye_results_cache_[i] = new_eye_left;
+                            m_react_right_eye_results_cache_[i] = new_eye_right;
+                        }
+                        
+                    } else {
+                        INSPIRE_LOGD("Serialized objects cannot connect to trace objects in memory, and there may be some problems");
+                    }
+                } else {
+                    INSPIRE_LOGW("The index of the trace object does not match the trace list in memory, and there may be some problems");
+                }
+            }
+        }
 
     }
 
@@ -212,6 +248,13 @@ const std::vector<float>& FaceContext::GetFaceQualityScoresResultsCache() const
     return m_quality_score_results_cache_;
 }
 
+const std::vector<float>& FaceContext::GetFaceInteractionLeftEyeStatusCache() const {
+    return m_react_left_eye_results_cache_;
+}
+
+const std::vector<float>& FaceContext::GetFaceInteractionRightEyeStatusCache() const {
+    return m_react_right_eye_results_cache_;
+}
 
 const Embedded& FaceContext::GetFaceFeatureCache() const {
     return m_face_feature_cache_;

cpp/inspireface/face_context.h

@@ -232,6 +232,18 @@ class INSPIRE_API FaceContext {
      */
     const std::vector<float>& GetFaceQualityScoresResultsCache() const;
 
+    /**
+     * @brief Gets the cache of left eye status predict results.
+     * @return A const reference to a vector containing eye status predict results.
+     */
+    const std::vector<float>& GetFaceInteractionLeftEyeStatusCache() const;
+
+    /**
+     * @brief Gets the cache of right eye status predict results.
+     * @return A const reference to a vector containing eye status predict results.
+     */
+    const std::vector<float>& GetFaceInteractionRightEyeStatusCache() const;
+
     /**
      * @brief Gets the cache of the current face features.
      * @return A const reference to the Embedded object containing current face feature data.
@@ -263,6 +275,8 @@ class INSPIRE_API FaceContext {
     std::vector<float> m_mask_results_cache_;                       ///< Cache for mask detection results
     std::vector<float> m_rgb_liveness_results_cache_;               ///< Cache for RGB liveness detection results
     std::vector<float> m_quality_score_results_cache_;               ///< Cache for RGB face quality score results
+    std::vector<float> m_react_left_eye_results_cache_;               ///< Cache for Left eye state in face interaction
+    std::vector<float> m_react_right_eye_results_cache_;               ///< Cache for Right eye state in face interaction
     Embedded m_face_feature_cache_;                                ///< Cache for current face feature data
 
     std::mutex m_mtx_;                                             ///< Mutex for thread safety.