1+ #ifndef INSPIRSE_FACE_FACE_ACTION_H
2+ #define INSPIRSE_FACE_FACE_ACTION_H
3+
4+ #include < iostream>
5+ #include " opencv2/opencv.hpp"
6+ #include " middleware/utils.h"
7+ #include " data_type.h"
8+ #include " track_module/landmark/face_landmark.h"
9+
10+ namespace inspire {
11+
12+ enum FACE_ACTION {
13+ NORMAL = 0 ,
14+ SHAKE = 0 ,
15+ BLINK = 1 ,
16+ JAW_OPEN = 2 ,
17+ RAISE_HEAD = 3
18+ };
19+
20+ typedef struct FaceActions {
21+ int normal = 0 ;
22+ int shake = 0 ;
23+ int blink = 0 ;
24+ int jawOpen = 0 ;
25+ int raiseHead = 0 ;
26+ } FaceActions;
27+
28+ class INSPIRE_API FaceActionAnalyse {
29+ public:
30+ FaceActionAnalyse (int record_list_length) {
31+ record_list.resize (record_list_length);
32+ record_list_euler.resize (record_list_length);
33+ record_list_eyes.resize (record_list_length);
34+ record_size = record_list_length;
35+ index = 0 ;
36+ }
37+
38+ void RecordActionFrame (const std::vector<cv::Point2f> &landmark,
39+ const cv::Vec3f &euler_angle,
40+ const cv::Vec2f &eyes_status) {
41+ MoveRecordList ();
42+ record_list[0 ] = landmark;
43+ record_list_euler[0 ] = euler_angle;
44+ record_list_eyes[0 ] = eyes_status;
45+ index += 1 ;
46+ }
47+
48+ void Reset () {
49+ record_list.clear ();
50+ record_list.resize (record_size);
51+ record_list_euler.clear ();
52+ record_list_euler.resize (record_size);
53+ record_list_eyes.clear ();
54+ record_list_eyes.resize (record_size);
55+ index = 0 ;
56+ }
57+
58+ FaceActions AnalysisFaceAction () {
59+ FaceActions actionRecord;
60+ actions.clear ();
61+ eye_state_list.clear ();
62+ if (index < record_list.size ()) {
63+ actions.push_back (NORMAL);
64+ actionRecord.normal = 1 ;
65+ } else {
66+ for (int i = 0 ; i < record_list_eyes.size (); i++) {
67+ const auto &eye = record_list_eyes[i];
68+ std::pair<float , float > eye_state (eye[0 ], eye[1 ]);
69+ eye_state_list.push_back (eye_state);
70+ }
71+
72+ // count mouth aspect ratio
73+ float mouth_widthwise_d =
74+ PointDistance (record_list[0 ][FaceLandmark::MOUTH_LEFT_CORNER],
75+ record_list[0 ][FaceLandmark::MOUTH_RIGHT_CORNER]);
76+ float mouth_heightwise_d =
77+ PointDistance (record_list[0 ][FaceLandmark::MOUTH_UPPER],
78+ record_list[0 ][FaceLandmark::MOUTH_LOWER]);
79+ float mouth_aspect_ratio = mouth_heightwise_d / mouth_widthwise_d;
80+ if (mouth_aspect_ratio > 0.3 ) {
81+ actions.push_back (JAW_OPEN);
82+ actionRecord.jawOpen = 1 ;
83+ }
84+
85+ int counter_eye_open = 0 ;
86+ int counter_eye_close = 0 ;
87+ for (auto &e : eye_state_list) {
88+ if (e.first < 0.5 || e.second < 0.5 ) {
89+ counter_eye_close += 1 ;
90+ }
91+ if (e.first > 0.5 || e.second > 0.5 ) {
92+ counter_eye_open += 1 ;
93+ }
94+ }
95+ if (counter_eye_close > 0 && counter_eye_open > 2 &&
96+ record_list_euler[0 ][1 ] > -6 && record_list_euler[0 ][0 ] < 6 ) {
97+ actions.push_back (BLINK);
98+ actionRecord.blink = 1 ;
99+ Reset ();
100+ }
101+
102+ bool counter_head_shake_left = false ;
103+ bool counter_head_shake_right = false ;
104+ for (auto &e : record_list_euler) {
105+ if (e[1 ] < -6 ) {
106+ counter_head_shake_left = true ;
107+ }
108+ if (e[1 ] > 6 ) {
109+ counter_head_shake_right = true ;
110+ }
111+ }
112+ if (counter_head_shake_left && counter_head_shake_right) {
113+ actions.push_back (SHAKE);
114+ actionRecord.shake = 1 ;
115+ }
116+
117+ if (record_list_euler[0 ][0 ] > 10 ) {
118+ actions.push_back (RAISE_HEAD);
119+ actionRecord.raiseHead = 1 ;
120+ }
121+
122+ }
123+ return actionRecord;
124+ }
125+
126+ std::vector<FACE_ACTION> GetActions () const {
127+ return actions;
128+ }
129+
130+ private:
131+ void MoveRecordList () {
132+ // for(int i = 0 ; i < record_list.size() - 1 ; i++){
133+ // record_list[i+1] = record_list[i];
134+ // record_list_euler[i+1] = record_list_euler[i];
135+ // }
136+ for (int i = record_list.size () - 1 ; i > 0 ; i--) {
137+ record_list[i] = record_list[i - 1 ];
138+ record_list_euler[i] = record_list_euler[i - 1 ];
139+ record_list_eyes[i] = record_list_eyes[i - 1 ];
140+ }
141+ }
142+
143+
144+ std::vector<std::vector<cv::Point2f>> record_list;
145+ std::vector<cv::Vec3f> record_list_euler;
146+ std::vector<cv::Vec2f> record_list_eyes;
147+ std::vector<std::pair<float , float >> eye_state_list; // pair left right
148+ std::vector<float > mouth_state_list;
149+ std::vector<FACE_ACTION> actions;
150+ int record_size;
151+ int index;
152+ };
153+
154+ } // namespace inspire
155+
156+ #endif
0 commit comments