Add gesture classification for handpose estimation (#168)

Author: WanliZhong

models/handpose_estimation_mediapipe/README.md

@@ -4,6 +4,9 @@ This model estimates 21 hand keypoints per detected hand from [palm detector](..
 
 ![MediaPipe Hands Keypoints](./example_outputs/hand_keypoints.png)
 
+Hand gesture classification demo (0-9)
+![hand gestures](./example_outputs/gesture_classification.png)
+
 This model is converted from TFlite to ONNX using following tools:
 - TFLite model to ONNX: https://github.com/onnx/tensorflow-onnx
 - simplified by [onnx-simplifier](https://github.com/daquexian/onnx-simplifier)

models/handpose_estimation_mediapipe/demo.py

@@ -85,6 +85,9 @@ def draw_lines(image, landmarks, is_draw_point=True, thickness=2):
             for p in landmarks:
                 cv.circle(image, p, thickness, (0, 0, 255), -1)
 
+    # used for gesture classification
+    gc = GestureClassification()
+
     for idx, handpose in enumerate(hands):
         conf = handpose[-1]
         bbox = handpose[0:4].astype(np.int32)
@@ -96,11 +99,14 @@ def draw_lines(image, landmarks, is_draw_point=True, thickness=2):
         landmarks_screen = handpose[4:67].reshape(21, 3).astype(np.int32)
         landmarks_word = handpose[67:130].reshape(21, 3)
 
+        gesture = gc.classify(landmarks_screen)
+
         # Print results
         if print_result:
             print('-----------hand {}-----------'.format(idx + 1))
             print('conf: {:.2f}'.format(conf))
             print('handedness: {}'.format(handedness_text))
+            print('gesture: {}'.format(gesture))
             print('hand box: {}'.format(bbox))
             print('hand landmarks: ')
             for l in landmarks_screen:
@@ -113,6 +119,8 @@ def draw_lines(image, landmarks, is_draw_point=True, thickness=2):
         cv.rectangle(display_screen, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 255, 0), 2)
         # draw handedness
         cv.putText(display_screen, '{}'.format(handedness_text), (bbox[0], bbox[1] + 12), cv.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 255))
+        # draw gesture
+        cv.putText(display_screen, '{}'.format(gesture), (bbox[0], bbox[1] + 30), cv.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 255))
         # Draw line between each key points
         landmarks_xy = landmarks_screen[:, 0:2]
         draw_lines(display_screen, landmarks_xy, is_draw_point=False)
@@ -149,6 +157,118 @@ def draw_lines(image, landmarks, is_draw_point=True, thickness=2):
 
     return display_screen, display_3d
 
+class GestureClassification:
+    def _vector_2_angle(self, v1, v2):
+        uv1 = v1 / np.linalg.norm(v1)
+        uv2 = v2 / np.linalg.norm(v2)
+        angle = np.degrees(np.arccos(np.dot(uv1, uv2)))
+        return angle
+
+    def _hand_angle(self, hand):
+        angle_list = []
+        # thumb
+        angle_ = self._vector_2_angle(
+            np.array([hand[0][0] - hand[2][0], hand[0][1] - hand[2][1]]),
+            np.array([hand[3][0] - hand[4][0], hand[3][1] - hand[4][1]])
+        )
+        angle_list.append(angle_)
+        # index
+        angle_ = self._vector_2_angle(
+            np.array([hand[0][0] - hand[6][0], hand[0][1] - hand[6][1]]),
+            np.array([hand[7][0] - hand[8][0], hand[7][1] - hand[8][1]])
+        )
+        angle_list.append(angle_)
+        # middle
+        angle_ = self._vector_2_angle(
+            np.array([hand[0][0] - hand[10][0], hand[0][1] - hand[10][1]]),
+            np.array([hand[11][0] - hand[12][0], hand[11][1] - hand[12][1]])
+        )
+        angle_list.append(angle_)
+        # ring
+        angle_ = self._vector_2_angle(
+            np.array([hand[0][0] - hand[14][0], hand[0][1] - hand[14][1]]),
+            np.array([hand[15][0] - hand[16][0], hand[15][1] - hand[16][1]])
+        )
+        angle_list.append(angle_)
+        # pink
+        angle_ = self._vector_2_angle(
+            np.array([hand[0][0] - hand[18][0], hand[0][1] - hand[18][1]]),
+            np.array([hand[19][0] - hand[20][0], hand[19][1] - hand[20][1]])
+        )
+        angle_list.append(angle_)
+        return angle_list
+
+    def _finger_status(self, lmList):
+        fingerList = []
+        originx, originy = lmList[0]
+        keypoint_list = [[5, 4], [6, 8], [10, 12], [14, 16], [18, 20]]
+        for point in keypoint_list:
+            x1, y1 = lmList[point[0]]
+            x2, y2 = lmList[point[1]]
+            if np.hypot(x2 - originx, y2 - originy) > np.hypot(x1 - originx, y1 - originy):
+                fingerList.append(True)
+            else:
+                fingerList.append(False)
+
+        return fingerList
+
+    def _classify(self, hand):
+        thr_angle = 65.
+        thr_angle_thumb = 30.
+        thr_angle_s = 49.
+        gesture_str = "Undefined"
+
+        angle_list = self._hand_angle(hand)
+
+        thumbOpen, firstOpen, secondOpen, thirdOpen, fourthOpen = self._finger_status(hand)
+        # Number
+        if (angle_list[0] > thr_angle_thumb) and (angle_list[1] > thr_angle) and (angle_list[2] > thr_angle) and (
+                angle_list[3] > thr_angle) and (angle_list[4] > thr_angle) and \
+                not firstOpen and not secondOpen and not thirdOpen and not fourthOpen:
+            gesture_str = "Zero"
+        elif (angle_list[0] > thr_angle_thumb) and (angle_list[1] < thr_angle_s) and (angle_list[2] > thr_angle) and (
+                angle_list[3] > thr_angle) and (angle_list[4] > thr_angle) and \
+                firstOpen and not secondOpen and not thirdOpen and not fourthOpen:
+            gesture_str = "One"
+        elif (angle_list[0] > thr_angle_thumb) and (angle_list[1] < thr_angle_s) and (angle_list[2] < thr_angle_s) and (
+                angle_list[3] > thr_angle) and (angle_list[4] > thr_angle) and \
+                not thumbOpen and firstOpen and secondOpen and not thirdOpen and not fourthOpen:
+            gesture_str = "Two"
+        elif (angle_list[0] > thr_angle_thumb) and (angle_list[1] < thr_angle_s) and (angle_list[2] < thr_angle_s) and (
+                angle_list[3] < thr_angle_s) and (angle_list[4] > thr_angle) and \
+                not thumbOpen and firstOpen and secondOpen and thirdOpen and not fourthOpen:
+            gesture_str = "Three"
+        elif (angle_list[0] > thr_angle_thumb) and (angle_list[1] < thr_angle_s) and (angle_list[2] < thr_angle_s) and (
+                angle_list[3] < thr_angle_s) and (angle_list[4] < thr_angle) and \
+                firstOpen and secondOpen and thirdOpen and fourthOpen:
+            gesture_str = "Four"
+        elif (angle_list[0] < thr_angle_s) and (angle_list[1] < thr_angle_s) and (angle_list[2] < thr_angle_s) and (
+                angle_list[3] < thr_angle_s) and (angle_list[4] < thr_angle_s) and \
+                thumbOpen and firstOpen and secondOpen and thirdOpen and fourthOpen:
+            gesture_str = "Five"
+        elif (angle_list[0] < thr_angle_s) and (angle_list[1] > thr_angle) and (angle_list[2] > thr_angle) and (
+                angle_list[3] > thr_angle) and (angle_list[4] < thr_angle_s) and \
+                thumbOpen and not firstOpen and not secondOpen and not thirdOpen and fourthOpen:
+            gesture_str = "Six"
+        elif (angle_list[0] < thr_angle_s) and (angle_list[1] < thr_angle) and (angle_list[2] > thr_angle) and (
+                angle_list[3] > thr_angle) and (angle_list[4] > thr_angle_s) and \
+                thumbOpen and firstOpen and not secondOpen and not thirdOpen and not fourthOpen:
+            gesture_str = "Seven"
+        elif (angle_list[0] < thr_angle_s) and (angle_list[1] < thr_angle) and (angle_list[2] < thr_angle) and (
+                angle_list[3] > thr_angle) and (angle_list[4] > thr_angle_s) and \
+                thumbOpen and firstOpen and secondOpen and not thirdOpen and not fourthOpen:
+            gesture_str = "Eight"
+        elif (angle_list[0] < thr_angle_s) and (angle_list[1] < thr_angle) and (angle_list[2] < thr_angle) and (
+                angle_list[3] < thr_angle) and (angle_list[4] > thr_angle_s) and \
+                thumbOpen and firstOpen and secondOpen and thirdOpen and not fourthOpen:
+            gesture_str = "Nine"
+
+        return gesture_str
+
+    def classify(self, landmarks):
+        hand = landmarks[:21, :2]
+        gesture = self._classify(hand)
+        return gesture
 
 if __name__ == '__main__':
     backend_id = backend_target_pairs[args.backend_target][0]