Add face actions

tunmx · tunmx · commit 4818ecdbc351 · 2024-07-07T23:20:20.000+08:00
Former-commit-id: de2ea53
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -9,7 +9,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
 # Current version
 set(INSPIRE_FACE_VERSION_MAJOR 1)
 set(INSPIRE_FACE_VERSION_MINOR 1)
-set(INSPIRE_FACE_VERSION_PATCH 4)
+set(INSPIRE_FACE_VERSION_PATCH 5)
 
 # Converts the version number to a string
 string(CONCAT INSPIRE_FACE_VERSION_MAJOR_STR ${INSPIRE_FACE_VERSION_MAJOR})
diff --git a/README.md b/README.md
@@ -13,6 +13,8 @@ Please contact [contact@insightface.ai](mailto:contact@insightface.ai?subject=In
 
 ## Change Logs
 
+**`2024-07-07`** Add some face action detection to the face interaction module.
+
 **`2024-07-05`** Fixed some bugs in the python ctypes interface.
 
 **`2024-07-03`** Add the blink detection algorithm of face interaction module.
diff --git a/cpp/inspireface/c_api/inspireface.cc b/cpp/inspireface/c_api/inspireface.cc
@@ -500,9 +500,14 @@ HResult HFMultipleFacePipelineProcess(HFSession session, HFImageStream streamHan
     if (stream == nullptr) {
         return HERR_INVALID_IMAGE_STREAM_HANDLE;
     }
-    if (faces->detectedNum <= 0 || faces->tokens->data == nullptr) {
+    if (faces->tokens->data == nullptr) {
         return HERR_INVALID_FACE_LIST;
     }
+
+    if (faces->detectedNum == 0) {
+        return HSUCCEED;
+    }
+
     inspire::ContextCustomParameter param;
     param.enable_mask_detect = parameter.enable_mask_detect;
     param.enable_face_attribute = parameter.enable_face_quality;
@@ -545,9 +550,12 @@ HResult HFMultipleFacePipelineProcessOptional(HFSession session, HFImageStream s
     if (stream == nullptr) {
         return HERR_INVALID_IMAGE_STREAM_HANDLE;
     }
-    if (faces->detectedNum <= 0 || faces->tokens->data == nullptr) {
+    if (faces->tokens->data == nullptr) {
         return HERR_INVALID_FACE_LIST;
     }
+    if (faces->detectedNum == 0) {
+        return HSUCCEED;
+    }
 
     inspire::ContextCustomParameter param;
     if (customOption & HF_ENABLE_FACE_RECOGNITION) {
@@ -572,7 +580,6 @@ HResult HFMultipleFacePipelineProcessOptional(HFSession session, HFImageStream s
         param.enable_interaction_liveness = true;
     } 
 
-
     HResult ret;
     std::vector<inspire::HyperFaceData> data;
     data.resize(faces->detectedNum);
@@ -654,8 +661,8 @@ HResult HFFaceQualityDetect(HFSession session, HFFaceBasicToken singleFace, HFlo
 
 }
 
-HResult HFGetFaceIntereactionResult(HFSession session, PHFFaceIntereactionResult result) {
-     if (session == nullptr) {
+HResult HFGetFaceIntereactionStateResult(HFSession session, PHFFaceIntereactionState result) {
+    if (session == nullptr) {
         return HERR_INVALID_CONTEXT_HANDLE;
     }
     HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession* ) session;
@@ -669,6 +676,24 @@ HResult HFGetFaceIntereactionResult(HFSession session, PHFFaceIntereactionResult
     return HSUCCEED;
 }
 
+HResult HFGetFaceIntereactionActionsResult(HFSession session, PHFFaceIntereactionsActions actions) {
+    if (session == nullptr) {
+        return HERR_INVALID_CONTEXT_HANDLE;
+    }
+    HF_FaceAlgorithmSession *ctx = (HF_FaceAlgorithmSession* ) session;
+    if (ctx == nullptr) {
+        return HERR_INVALID_CONTEXT_HANDLE;
+    }
+    actions->num = ctx->impl.GetFaceNormalAactionsResultCache().size();
+    actions->normal = (HInt32* )ctx->impl.GetFaceNormalAactionsResultCache().data();
+    actions->blink = (HInt32* )ctx->impl.GetFaceBlinkAactionsResultCache().data();
+    actions->shake = (HInt32* )ctx->impl.GetFaceShakeAactionsResultCache().data();
+    actions->headRiase = (HInt32* )ctx->impl.GetFaceRaiseHeadAactionsResultCache().data();
+    actions->jawOpen = (HInt32* )ctx->impl.GetFaceJawOpenAactionsResultCache().data();
+
+    return HSUCCEED;
+}
+
 HResult HFGetFaceAttributeResult(HFSession session, PHFFaceAttributeResult results) {
     if (session == nullptr) {
         return HERR_INVALID_CONTEXT_HANDLE;
diff --git a/cpp/inspireface/c_api/inspireface.h b/cpp/inspireface/c_api/inspireface.h
@@ -638,19 +638,38 @@ HYPER_CAPI_EXPORT extern HResult HFFaceQualityDetect(HFSession session, HFFaceBa
 /**
  * @brief Facial states in the face interaction module.
  */
-typedef struct HFFaceIntereactionResult {
+typedef struct HFFaceIntereactionState {
     HInt32 num;                             ///< Number of faces detected.
     HPFloat leftEyeStatusConfidence;        ///< Left eye state: confidence close to 1 means open, close to 0 means closed.
     HPFloat rightEyeStatusConfidence;       ///< Right eye state: confidence close to 1 means open, close to 0 means closed.
-} HFFaceIntereactionResult, *PHFFaceIntereactionResult;
+} HFFaceIntereactionState, *PHFFaceIntereactionState;
 
 /**
  * @brief Get the prediction results of face interaction.
  * @param session Handle to the session.
  * @param result Facial state prediction results in the face interaction module.
  */
-HYPER_CAPI_EXPORT extern HResult HFGetFaceIntereactionResult(HFSession session, PHFFaceIntereactionResult result);
+HYPER_CAPI_EXPORT extern HResult HFGetFaceIntereactionStateResult(HFSession session, PHFFaceIntereactionState result);
 
+/**
+ * @brief Actions detected in the face interaction module.
+ */
+typedef struct HFFaceIntereactionsActions {
+    HInt32 num;          ///< Number of actions detected.
+    HPInt32 normal;      ///< Normal actions.
+    HPInt32 shake;       ///< Shake actions.
+    HPInt32 jawOpen;     ///< Jaw open actions.
+    HPInt32 headRiase;   ///< Head raise actions.
+    HPInt32 blink;       ///< Blink actions.
+} HFFaceIntereactionsActions, *PHFFaceIntereactionsActions;
+
+/**
+ * @brief Get the prediction results of face interaction actions.
+ * @param session Handle to the session.
+ * @param actions Facial action prediction results in the face interaction module.
+ * @return HResult indicating success or failure of the function call.
+ */
+HYPER_CAPI_EXPORT extern HResult HFGetFaceIntereactionActionsResult(HFSession session, PHFFaceIntereactionsActions actions);
 /**
  * @brief Struct representing face attribute results.
  *
diff --git a/cpp/inspireface/common/face_info/face_action.h b/cpp/inspireface/common/face_info/face_action.h
@@ -12,12 +12,19 @@ namespace inspire {
 enum FACE_ACTION {
   NORMAL = 0,
   SHAKE = 0,
-  BROW_UP = 1,
-  BLINK = 2,
-  JAW_OPEN = 3,
-  HEAD_RISE = 4
+  BLINK = 1,
+  JAW_OPEN = 2,
+  RAISE_HEAD = 3
 };
 
+typedef struct FaceActions{
+    int normal = 0;
+    int shake = 0;
+    int blink = 0;
+    int jawOpen = 0;
+    int raiseHead = 0;
+} FaceActions;
+
 class INSPIRE_API FaceActionAnalyse {
 public:
     FaceActionAnalyse(int record_list_length) {
@@ -48,11 +55,13 @@ class INSPIRE_API FaceActionAnalyse {
         index = 0;
     } 
 
-    void AnalysisFaceAction() {
+    FaceActions AnalysisFaceAction() {
+        FaceActions actionRecord;
         actions.clear();
         eye_state_list.clear();
         if (index < record_list.size()) {
             actions.push_back(NORMAL);
+            actionRecord.normal = 1;
         } else {
             for (int i = 0; i < record_list_eyes.size(); i++) {
                 const auto &eye = record_list_eyes[i];
@@ -70,6 +79,7 @@ class INSPIRE_API FaceActionAnalyse {
             float mouth_aspect_ratio = mouth_heightwise_d / mouth_widthwise_d;
             if (mouth_aspect_ratio > 0.3) {
                 actions.push_back(JAW_OPEN);
+                actionRecord.jawOpen = 1;
             }
 
             int counter_eye_open = 0;
@@ -85,6 +95,7 @@ class INSPIRE_API FaceActionAnalyse {
             if (counter_eye_close > 0 && counter_eye_open > 2 &&
                 record_list_euler[0][1] > -6 && record_list_euler[0][0] < 6) {
                 actions.push_back(BLINK);
+                actionRecord.blink = 1;
                 Reset();
             }
 
@@ -100,13 +111,20 @@ class INSPIRE_API FaceActionAnalyse {
             }
             if (counter_head_shake_left && counter_head_shake_right) {
                 actions.push_back(SHAKE);
+                actionRecord.shake = 1;
             }
 
-            if (record_list_euler[0][0] < -10) {
-                actions.push_back(HEAD_RISE);
+            if (record_list_euler[0][0] > 10) {
+                actions.push_back(RAISE_HEAD);
+                actionRecord.raiseHead = 1;
             }
 
         }
+        return actionRecord;
+    }
+
+    std::vector<FACE_ACTION> GetActions() const {
+        return actions;
     }
 
 private:
diff --git a/cpp/inspireface/common/face_info/face_object.h b/cpp/inspireface/common/face_info/face_object.h
@@ -138,11 +138,11 @@ class INSPIRE_API FaceObject {
         return box_square;
     }
 
-    void UpdateFaceAction() {
+    FaceActions UpdateFaceAction() {
         cv::Vec3f euler(high_result.pitch, high_result.yaw, high_result.roll);
         cv::Vec2f eyes(left_eye_status_.back(), right_eye_status_.back());
         face_action_->RecordActionFrame(landmark_, euler, eyes);
-        face_action_->AnalysisFaceAction();
+        return face_action_->AnalysisFaceAction();
     }
 
     void DisableTracking() { tracking_state_ = UNTRACKING; }
diff --git a/cpp/inspireface/face_context.cpp b/cpp/inspireface/face_context.cpp
@@ -63,6 +63,13 @@ int32_t FaceContext::FaceDetectAndTrack(CameraStream &image) {
     m_quality_score_results_cache_.clear();
     m_react_left_eye_results_cache_.clear();
     m_react_right_eye_results_cache_.clear();
+
+    m_action_normal_results_cache_.clear();
+    m_action_shake_results_cache_.clear();
+    m_action_blink_results_cache_.clear();
+    m_action_jaw_open_results_cache_.clear();
+    m_action_raise_head_results_cache_.clear();
+
     m_quality_score_results_cache_.clear();
     m_attribute_race_results_cache_.clear();
     m_attribute_gender_results_cache_.clear();
@@ -138,6 +145,11 @@ int32_t FaceContext::FacesProcess(CameraStream &image, const std::vector<HyperFa
     m_attribute_race_results_cache_.resize(faces.size(), -1);
     m_attribute_gender_results_cache_.resize(faces.size(), -1);
     m_attribute_age_results_cache_.resize(faces.size(), -1);
+    m_action_normal_results_cache_.resize(faces.size(), -1);
+    m_action_jaw_open_results_cache_.resize(faces.size(), -1);
+    m_action_blink_results_cache_.resize(faces.size(), -1);
+    m_action_raise_head_results_cache_.resize(faces.size(), -1);
+    m_action_shake_results_cache_.resize(faces.size(), -1);
     for (int i = 0; i < faces.size(); ++i) {
         const auto &face = faces[i];
         // RGB Liveness Detect
@@ -190,8 +202,12 @@ int32_t FaceContext::FacesProcess(CameraStream &image, const std::vector<HyperFa
                             m_react_left_eye_results_cache_[i] = new_eye_left;
                             m_react_right_eye_results_cache_[i] = new_eye_right;
                         }
-                        target.UpdateFaceAction();
-                        
+                        const auto actions = target.UpdateFaceAction();
+                        m_action_normal_results_cache_[i] = actions.normal;
+                        m_action_jaw_open_results_cache_[i] = actions.jawOpen;
+                        m_action_blink_results_cache_[i] = actions.blink;
+                        m_action_raise_head_results_cache_[i] = actions.raiseHead;
+                        m_action_shake_results_cache_[i] = actions.shake;
                     } else {
                         INSPIRE_LOGD("Serialized objects cannot connect to trace objects in memory, and there may be some problems");
                     }
@@ -275,6 +291,26 @@ const std::vector<int>& FaceContext::GetFaceAgeBracketResultsCache() const {
     return m_attribute_age_results_cache_;
 }
 
+const std::vector<int>& FaceContext::GetFaceNormalAactionsResultCache() const {
+    return m_action_normal_results_cache_;
+}
+
+const std::vector<int>& FaceContext::GetFaceJawOpenAactionsResultCache() const {
+    return m_action_jaw_open_results_cache_;
+}
+
+const std::vector<int>& FaceContext::GetFaceBlinkAactionsResultCache() const {
+    return m_action_blink_results_cache_;
+}
+
+const std::vector<int>& FaceContext::GetFaceShakeAactionsResultCache() const {
+    return m_action_shake_results_cache_;
+}
+
+const std::vector<int>& FaceContext::GetFaceRaiseHeadAactionsResultCache() const {
+    return m_action_raise_head_results_cache_;
+}
+
 int32_t FaceContext::FaceFeatureExtract(CameraStream &image, FaceBasicData& data) {
     std::lock_guard<std::mutex> lock(m_mtx_);
     int32_t ret;
diff --git a/cpp/inspireface/face_context.h b/cpp/inspireface/face_context.h
@@ -2,6 +2,7 @@
 // Created by Tunm-Air13 on 2023/9/7.
 //
 #pragma once
+#include <vector>
 #ifndef HYPERFACEREPO_FACE_CONTEXT_H
 #define HYPERFACEREPO_FACE_CONTEXT_H
 
@@ -261,6 +262,36 @@ class INSPIRE_API FaceContext {
      */
     const std::vector<int>& GetFaceAgeBracketResultsCache() const;
 
+    /**
+     * @brief Gets the cache of face action normal results.
+     * @return A const reference to a vector containing face action normal results.
+     */
+    const std::vector<int>& GetFaceNormalAactionsResultCache() const;
+    
+    /**
+     * @brief Gets the cache of face action jaw open results.
+     * @return A const reference to a vector containing face action jaw open results.
+     */
+    const std::vector<int>& GetFaceJawOpenAactionsResultCache() const;
+
+    /**
+     * @brief Gets the cache of face action blink results.
+     * @return A const reference to a vector containing face action blink results.
+     */
+    const std::vector<int>& GetFaceBlinkAactionsResultCache() const;
+
+    /**
+     * @brief Gets the cache of face action shake results.
+     * @return A const reference to a vector containing face action shake results.
+     */
+    const std::vector<int>& GetFaceShakeAactionsResultCache() const;
+
+    /**
+     * @brief Gets the cache of face action raise head results.
+     * @return A const reference to a vector containing face action raise head results.
+     */
+    const std::vector<int>& GetFaceRaiseHeadAactionsResultCache() const;
+
     /**
      * @brief Gets the cache of the current face features.
      * @return A const reference to the Embedded object containing current face feature data.
@@ -294,6 +325,13 @@ class INSPIRE_API FaceContext {
     std::vector<float> m_quality_score_results_cache_;               ///< Cache for RGB face quality score results
     std::vector<float> m_react_left_eye_results_cache_;               ///< Cache for Left eye state in face interaction
     std::vector<float> m_react_right_eye_results_cache_;               ///< Cache for Right eye state in face interaction
+
+    std::vector<int> m_action_normal_results_cache_;                ///< Cache for normal action in face interaction
+    std::vector<int> m_action_shake_results_cache_;                 ///< Cache for shake action in face interaction
+    std::vector<int> m_action_blink_results_cache_;                 ///< Cache for blink action in face interaction
+    std::vector<int> m_action_jaw_open_results_cache_;              ///< Cache for jaw open action in face interaction
+    std::vector<int> m_action_raise_head_results_cache_;            ///< Cache for raise head action in face interaction
+
     std::vector<int> m_attribute_race_results_cache_;
     std::vector<int> m_attribute_gender_results_cache_;
     std::vector<int> m_attribute_age_results_cache_;
diff --git a/cpp/inspireface/information.h b/cpp/inspireface/information.h
@@ -7,6 +7,6 @@
 
 #define INSPIRE_FACE_VERSION_MAJOR_STR "1"
 #define INSPIRE_FACE_VERSION_MINOR_STR "1"
-#define INSPIRE_FACE_VERSION_PATCH_STR "4"
+#define INSPIRE_FACE_VERSION_PATCH_STR "5"
 
 #endif //HYPERFACEREPO_INFORMATION_H
diff --git a/cpp/inspireface/track_module/face_track.cpp b/cpp/inspireface/track_module/face_track.cpp
@@ -307,15 +307,25 @@ void FaceTrack::nms(float th) {
             float inter = w * h;
             float ovr = inter / (area[i] + area[j] - inter);
             if (ovr >= th) {
-                trackingFace.erase(trackingFace.begin() + j);
-                area.erase(area.begin() + j);
+                // Compare tracking IDs to decide which to keep
+                if (trackingFace[i].GetTrackingId() < trackingFace[j].GetTrackingId()) {
+                    trackingFace.erase(trackingFace.begin() + j);
+                    area.erase(area.begin() + j);
+                } else {
+                    trackingFace.erase(trackingFace.begin() + i);
+                    area.erase(area.begin() + i);
+                    // If we erase i, we need to break the inner loop
+                    // and start over with a new i, because the indexes shifted.
+                    break;
+                }
             } else {
                 j++;
             }
         }
     }
 }
 
+
 void FaceTrack::BlackingTrackingRegion(cv::Mat &image, cv::Rect &rect_mask) {
     int height = image.rows;
     int width = image.cols;
diff --git a/cpp/inspireface/version.txt b/cpp/inspireface/version.txt
@@ -1 +1 @@
-InspireFace Version: 1.1.4
+InspireFace Version: 1.1.5
diff --git a/cpp/sample/cpp/sample_face_track_video.cpp b/cpp/sample/cpp/sample_face_track_video.cpp
diff --git a/cpp/test/unit/api/test_face_pipeline.cpp b/cpp/test/unit/api/test_face_pipeline.cpp
diff --git a/python/inspireface/modules/core/native.py b/python/inspireface/modules/core/native.py
diff --git a/python/inspireface/modules/inspire_face.py b/python/inspireface/modules/inspire_face.py
diff --git a/python/sample_face_track_from_video.py b/python/sample_face_track_from_video.py

Original file line number	Diff line number	Diff line change
`@@ -138,11 +138,11 @@ class INSPIRE_API FaceObject {`
`138`	`138`	`return box_square;`
`139`	`139`	`}`
`140`	`140`
`141`		`- void UpdateFaceAction() {`
	`141`	`+ FaceActions UpdateFaceAction() {`
`142`	`142`	`cv::Vec3f euler(high_result.pitch, high_result.yaw, high_result.roll);`
`143`	`143`	`cv::Vec2f eyes(left_eye_status_.back(), right_eye_status_.back());`
`144`	`144`	`face_action_->RecordActionFrame(landmark_, euler, eyes);`
`145`		`- face_action_->AnalysisFaceAction();`
	`145`	`+ return face_action_->AnalysisFaceAction();`
`146`	`146`	`}`
`147`	`147`
`148`	`148`	`void DisableTracking() { tracking_state_ = UNTRACKING; }`