Skip to content

Commit fe31584

Browse files
tunmxtunmx
committed
Add face actions
Former-commit-id: 135545b
1 parent 33299ce commit fe31584

File tree

6 files changed

+157
-7
lines changed

6 files changed

+157
-7
lines changed

‎cpp/inspireface/common/face_info/face_action.h‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
#ifndef INSPIRSE_FACE_FACE_ACTION_H
2+
#define INSPIRSE_FACE_FACE_ACTION_H
3+
4+
#include <iostream>
5+
#include "opencv2/opencv.hpp"
6+
#include "middleware/utils.h"
7+
#include "data_type.h"
8+
#include "track_module/landmark/face_landmark.h"
9+
10+
namespace inspire {
11+
12+
enum FACE_ACTION {
13+
NORMAL = 0,
14+
SHAKE = 0,
15+
BROW_UP = 1,
16+
BLINK = 2,
17+
JAW_OPEN = 3,
18+
HEAD_RISE = 4
19+
};
20+
21+
class INSPIRE_API FaceActionAnalyse {
22+
public:
23+
FaceActionAnalyse(int record_list_length) {
24+
record_list.resize(record_list_length);
25+
record_list_euler.resize(record_list_length);
26+
record_list_eyes.resize(record_list_length);
27+
record_size = record_list_length;
28+
index = 0;
29+
}
30+
31+
void RecordActionFrame(const std::vector<cv::Point2f> &landmark,
32+
const cv::Vec3f &euler_angle,
33+
const cv::Vec2f &eyes_status) {
34+
MoveRecordList();
35+
record_list[0] = landmark;
36+
record_list_euler[0] = euler_angle;
37+
record_list_eyes[0] = eyes_status;
38+
index += 1;
39+
}
40+
41+
void Reset() {
42+
record_list.clear();
43+
record_list.resize(record_size);
44+
record_list_euler.clear();
45+
record_list_euler.resize(record_size);
46+
record_list_eyes.clear();
47+
record_list_eyes.resize(record_size);
48+
index = 0;
49+
}
50+
51+
void AnalysisFaceAction() {
52+
actions.clear();
53+
eye_state_list.clear();
54+
if (index < record_list.size()) {
55+
actions.push_back(NORMAL);
56+
} else {
57+
for (int i = 0; i < record_list_eyes.size(); i++) {
58+
const auto &eye = record_list_eyes[i];
59+
std::pair<float, float> eye_state(eye[0], eye[1]);
60+
eye_state_list.push_back(eye_state);
61+
}
62+
63+
// count mouth aspect ratio
64+
float mouth_widthwise_d =
65+
PointDistance(record_list[0][FaceLandmark::MOUTH_LEFT_CORNER],
66+
record_list[0][FaceLandmark::MOUTH_RIGHT_CORNER]);
67+
float mouth_heightwise_d =
68+
PointDistance(record_list[0][FaceLandmark::MOUTH_UPPER],
69+
record_list[0][FaceLandmark::MOUTH_LOWER]);
70+
float mouth_aspect_ratio = mouth_heightwise_d / mouth_widthwise_d;
71+
if (mouth_aspect_ratio > 0.3) {
72+
actions.push_back(JAW_OPEN);
73+
}
74+
75+
int counter_eye_open = 0;
76+
int counter_eye_close = 0;
77+
for (auto &e : eye_state_list) {
78+
if (e.first < 0.5 || e.second < 0.5) {
79+
counter_eye_close += 1;
80+
}
81+
if (e.first > 0.5 || e.second > 0.5) {
82+
counter_eye_open += 1;
83+
}
84+
}
85+
if (counter_eye_close > 0 && counter_eye_open > 2 &&
86+
record_list_euler[0][1] > -6 && record_list_euler[0][0] < 6) {
87+
actions.push_back(BLINK);
88+
Reset();
89+
}
90+
91+
bool counter_head_shake_left = false;
92+
bool counter_head_shake_right = false;
93+
for (auto &e : record_list_euler) {
94+
if (e[1] < -6) {
95+
counter_head_shake_left = true;
96+
}
97+
if (e[1] > 6) {
98+
counter_head_shake_right = true;
99+
}
100+
}
101+
if (counter_head_shake_left && counter_head_shake_right) {
102+
actions.push_back(SHAKE);
103+
}
104+
105+
if (record_list_euler[0][0] < -10) {
106+
actions.push_back(HEAD_RISE);
107+
}
108+
109+
}
110+
}
111+
112+
private:
113+
void MoveRecordList() {
114+
// for(int i = 0 ; i < record_list.size() - 1 ; i++){
115+
// record_list[i+1] = record_list[i];
116+
// record_list_euler[i+1] = record_list_euler[i];
117+
//}
118+
for (int i = record_list.size() - 1; i > 0; i--) {
119+
record_list[i] = record_list[i - 1];
120+
record_list_euler[i] = record_list_euler[i - 1];
121+
record_list_eyes[i] = record_list_eyes[i - 1];
122+
}
123+
}
124+
125+
126+
std::vector<std::vector<cv::Point2f>> record_list;
127+
std::vector<cv::Vec3f> record_list_euler;
128+
std::vector<cv::Vec2f> record_list_eyes;
129+
std::vector<std::pair<float, float>> eye_state_list; // pair left right
130+
std::vector<float> mouth_state_list;
131+
std::vector<FACE_ACTION> actions;
132+
int record_size;
133+
int index;
134+
};
135+
136+
} // namespace inspire
137+
138+
#endif

‎cpp/inspireface/common/face_info/face_object.h‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "data_type.h"
1111
#include "face_process.h"
1212
#include "track_module/quality/face_pose_quality.h"
13+
#include "face_action.h"
1314

1415
namespace inspire {
1516

@@ -28,7 +29,7 @@ class INSPIRE_API FaceObject {
2829
tracking_count_ = 0;
2930
pose_euler_angle_.resize(3);
3031
keyPointFive.resize(5);
31-
// face_action_ = std::make_shared<FaceAction>(10);
32+
face_action_ = std::make_shared<FaceActionAnalyse>(10);
3233
}
3334

3435
void UpdateMatrix(const cv::Mat &matrix) {
@@ -138,8 +139,10 @@ class INSPIRE_API FaceObject {
138139
}
139140

140141
void UpdateFaceAction() {
141-
// face_action_->RecordActionFrame(landmark_, euler_angle_);
142-
// face_action_->AnalysisFaceAction();
142+
cv::Vec3f euler(high_result.pitch, high_result.yaw, high_result.roll);
143+
cv::Vec2f eyes(left_eye_status_.back(), right_eye_status_.back());
144+
face_action_->RecordActionFrame(landmark_, euler, eyes);
145+
face_action_->AnalysisFaceAction();
143146
}
144147

145148
void DisableTracking() { tracking_state_ = UNTRACKING; }
@@ -318,7 +321,7 @@ class INSPIRE_API FaceObject {
318321

319322
private:
320323
TRACK_STATE tracking_state_;
321-
// std::shared_ptr<FaceAction> face_action_;
324+
std::shared_ptr<FaceActionAnalyse> face_action_;
322325
int face_id_;
323326
};
324327

‎cpp/inspireface/face_context.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,7 @@ int32_t FaceContext::FacesProcess(CameraStream &image, const std::vector<HyperFa
190190
m_react_left_eye_results_cache_[i] = new_eye_left;
191191
m_react_right_eye_results_cache_[i] = new_eye_right;
192192
}
193+
target.UpdateFaceAction();
193194

194195
} else {
195196
INSPIRE_LOGD("Serialized objects cannot connect to trace objects in memory, and there may be some problems");

‎cpp/inspireface/track_module/face_track.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,6 @@ bool FaceTrack::TrackFace(CameraStream &image, FaceObject &face) {
230230
}
231231

232232
face.SetConfidence(score);
233-
face.UpdateFaceAction();
234233
return true;
235234
}
236235

‎cpp/inspireface/track_module/landmark/face_landmark.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ class INSPIRE_API FaceLandmark: public AnyNet {
4242
const static int NOSE_CORNER = 69; ///< Landmark index for the tip of the nose.
4343
const static int MOUTH_LEFT_CORNER = 45; ///< Landmark index for the left corner of the mouth.
4444
const static int MOUTH_RIGHT_CORNER = 50; ///< Landmark index for the right corner of the mouth.
45+
const static int MOUTH_LOWER = 37 - 1;
46+
const static int MOUTH_UPPER = 104 - 1;
4547

4648
const static int NUM_OF_LANDMARK = 106; ///< Total number of landmarks detected.
4749

‎python/sample_face_track_from_video.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ def case_face_tracker_from_video(resource_path, source, show, out):
5151
assert ret, "Launch failure. Please ensure the resource path is correct."
5252

5353
# Optional features, loaded during session creation based on the modules specified.
54-
opt = HF_ENABLE_NONE
55-
session = ifac.InspireFaceSession(opt, HF_DETECT_MODE_TRACK_BY_DETECTION, max_detect_num=25, detect_pixel_level=640) # Use video mode
54+
opt = HF_ENABLE_NONE | HF_ENABLE_INTERACTION
55+
session = ifac.InspireFaceSession(opt, HF_DETECT_MODE_LIGHT_TRACK, max_detect_num=25, detect_pixel_level=320) # Use video mode
5656
session.set_filter_minimum_face_pixel_size(0)
5757
# Determine if the source is a digital webcam index or a video file path.
5858
try:
@@ -85,6 +85,13 @@ def case_face_tracker_from_video(resource_path, source, show, out):
8585

8686
# Process frame here (e.g., face detection/tracking).
8787
faces = session.face_detection(frame)
88+
89+
# Features must be enabled during session creation to use them here.
90+
select_exec_func = HF_ENABLE_INTERACTION
91+
# Execute the pipeline to obtain richer face information.
92+
if len(faces) > 0:
93+
extends = session.face_pipeline(frame, faces, select_exec_func)
94+
8895
for idx, face in enumerate(faces):
8996
print(f"{'==' * 20}")
9097
print(f"idx: {idx}")

0 commit comments

Comments
 (0)