Facetracking and webcamera preview optimizations

Author: Sergey Popov

facetracker.py

@@ -5,7 +5,6 @@
 import traceback
 import gc
 import mediapipe as mp
-import pickle
 mp_hands = mp.solutions.hands
 
 # mp_drawing = mp.solutions.drawing_utils
@@ -55,7 +54,6 @@
 parser.add_argument("--dump-points", type=str, help="When set to a filename, the current face 3D points are made symmetric and dumped to the given file when quitting the visualization with the \"q\" key", default="")
 parser.add_argument("--benchmark", type=int, help="When set to 1, the different tracking models are benchmarked, starting with the best and ending with the fastest and with gaze tracking disabled for models with negative IDs", default=0)
 parser.add_argument("--frame-data", type=int, help="When set to 1, the server is sending webcam frame data", default=0)
-parser.add_argument("--webcam-preview", type=int, help="When set to 1, the server is pop ups webcam preview window", default=0)
 
 if os.name == 'nt':
     parser.add_argument("--use-dshowcapture", type=int, help="When set to 1, libdshowcapture will be used for video input instead of OpenCV", default=1)
@@ -133,7 +131,6 @@ def flush(self):
     sys.exit(0)
 
 import math
-import pickle
 import numpy as np
 import time
 import cv2
@@ -142,8 +139,6 @@ def flush(self):
 import json
 from input_reader import InputReader, VideoReader, DShowCaptureReader, try_int
 from tracker import Tracker, get_model_base_path
-import pyvirtualcam
-from pyvirtualcam import PixelFormat
 
 if args.benchmark > 0:
     model_base_path = get_model_base_path(args.model_dir)
@@ -161,7 +156,7 @@ def flush(self):
         print(1. / (total / 100.))
     sys.exit(0)
 
-max_length = 65535
+max_length = 65535-28 # 28 is for UDP header
 
 
 target_ip = args.ip
@@ -224,8 +219,6 @@ def flush(self):
     source_name = input_reader.name
     if args.hands == 1:
         holistic = mp_hands.Hands(model_complexity=1,min_detection_confidence=0.82,min_tracking_confidence=0.82)
-    if args.webcam_preview == 1:
-        cam =  pyvirtualcam.Camera(width, height, fps, fmt=PixelFormat.BGR, print_fps=fps)
 
     while repeat or input_reader.is_open(): 
         if not input_reader.is_open() or need_reinit == 1:
@@ -272,9 +265,6 @@ def flush(self):
             tracker = Tracker(width, height, threshold=args.threshold, max_threads=args.max_threads, max_faces=args.faces, discard_after=args.discard_after, scan_every=args.scan_every, silent=False if args.silent == 0 else True, model_type=args.model, model_dir=args.model_dir, no_gaze=False if args.gaze_tracking != 0 and args.model != -1 else True, detection_threshold=args.detection_threshold, use_retinaface=args.scan_retinaface, max_feature_updates=args.max_feature_updates, static_model=True if args.no_3d_adapt == 1 else False, try_hard=args.try_hard == 1)
             if args.video_out is not None:
                 out = cv2.VideoWriter(args.video_out, cv2.VideoWriter_fourcc('F','F','V','1'), args.video_fps, (width * args.video_scale, height * args.video_scale))
-        if args.webcam_preview == 1:
-            cam.send(frame)
-            cam.sleep_until_next_frame()
 
         try:
             inference_start = time.perf_counter()
@@ -428,8 +418,8 @@ def flush(self):
 
 
             if args.frame_data == 1:
-                # frame = frame if width < 720 else cv2.resize(frame, (720, 480), interpolation=cv2.INTER_NEAREST)
-                retval, buffer = cv2.imencode(".jpg", frame)
+                cam_frame = frame if width <= 480 else cv2.resize(frame, (480, math.ceil(height * (480 / width))), interpolation=cv2.INTER_NEAREST)
+                retval, buffer = cv2.imencode(".jpg", cam_frame)
                 if retval:
                     # convert to byte array
                     buffer = buffer.tobytes()
@@ -550,8 +540,6 @@ def flush(self):
         print("Quitting")
 if args.hands == 1 and holistic is not None:
     holistic.close()
-if args.webcam_preview == 1 and cam is not None:
-    cam.close()
 input_reader.close()
 if out is not None:
     out.release()

facetracker.spec

@@ -1,53 +1,50 @@
 # -*- mode: python ; coding: utf-8 -*-
 
-block_cipher = None
-
 
-a = Analysis(['facetracker.py'],
-             pathex=['C:\\OpenSeeFaceBuild'],
-             binaries=[('dshowcapture/dshowcapture_x86.dll', '.'), ('dshowcapture/dshowcapture_x64.dll', '.'), ('dshowcapture/libminibmcapture32.dll', '.'), ('dshowcapture/libminibmcapture64.dll', '.'), ('escapi/escapi_x86.dll', '.'), ('escapi/escapi_x64.dll', '.'), ('run.bat', '.'), ('msvcp140.dll', '.'), ('vcomp140.dll', '.'), ('concrt140.dll', '.'), ('vccorlib140.dll', '.')],
-             datas=[('mediapipe\modules\hand_landmark', 'mediapipe\modules\hand_landmark'), ('mediapipe\modules\palm_detection', 'mediapipe\modules\palm_detection')],
-             hiddenimports=[],
-             hookspath=[],
-             runtime_hooks=[],
-             excludes=['mpl-data', 'PyInstaller', 'pywt', 'skimage', 'scipy', 'pyinstaller'],
-             win_no_prefer_redirects=False,
-             win_private_assemblies=False,
-             cipher=block_cipher,
-             noarchive=False)
-
-remove_bin = []
-for bin in a.binaries:
-    if bin[0].startswith("opencv_video") or bin[0].startswith("PyInstaller"):
-        remove_bin.append(bin[0])
-        print(bin)
-remove_dat = []
-for bin in a.datas:
-    if bin[0].startswith("opencv_video") or bin[0].startswith("PyInstaller"):
-        remove_dat.append(bin[0])
-        print("data ", bin)
+block_cipher = None
 
-a.binaries = [x for x in a.binaries if not x[0] in remove_bin]
-a.datas = [x for x in a.datas if not x[0] in remove_dat]
 
-pyz = PYZ(a.pure, a.zipped_data,
-             cipher=block_cipher)
-exe = EXE(pyz,
-          a.scripts,
-          [],
-          exclude_binaries=True,
-          name='facetracker',
-          debug=False,
-          bootloader_ignore_signals=False,
-          strip=False,
-          upx=True,
-          console=True )
+a = Analysis(
+    ['facetracker.py'],
+    pathex=[],
+    binaries=[('dshowcapture/*.dll', '.'), ('escapi/*.dll', '.'), ('venv/lib/site-packages/onnxruntime/capi/*.dll', 'onnxruntime\\capi'), ('venv/lib/site-packages/mediapipe/python/*.dll', 'mediapipe\\python'), ('msvcp140.dll', '.'), ('vcomp140.dll', '.'), ('concrt140.dll', '.'), ('vccorlib140.dll', '.'), ('run.bat', '.')],
+    datas=[('mediapipe/modules/hand_landmark', '.'), ('mediapipe/modules/hand_landmark', '.'), ('mediapipe/modules/palm_detection', '.'), ('mediapipe/modules/palm_detection', '.')],
+    hiddenimports=[],
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
 
-coll = COLLECT(exe,
-               a.binaries,
-               a.zipfiles,
-               a.datas,
-               strip=False,
-               upx=True,
-               upx_exclude=[],
-               name='facetracker')
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='facetracker',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    name='facetracker',
+)

make_exe.bat

@@ -11,19 +11,24 @@ call venv\Scripts\activate.bat
 echo "Installing dependencies"
 pip install wheel
 
-pip install onnxruntime opencv-python==4.5.4.60 pillow numpy==1.23.0 pyinstaller
+pip install onnxruntime opencv-python==4.5.4.60 pillow numpy==1.23.0 pyinstaller mediapipe=0.10.2
 
 echo "Running pyinstaller"
 pyinstaller facetracker.py --clean ^
     --onedir ^
     --add-binary dshowcapture/*.dll;. ^
     --add-binary escapi/*.dll;. ^
     --add-binary venv/lib/site-packages/onnxruntime/capi/*.dll;onnxruntime\capi ^
+    --add-binary venv/lib/site-packages/mediapipe/python/*.dll;mediapipe\python ^
     --add-binary msvcp140.dll;. ^
     --add-binary vcomp140.dll;. ^
     --add-binary concrt140.dll;. ^
     --add-binary vccorlib140.dll;. ^
-    --add-binary run.bat;.
+    --add-binary run.bat;. ^
+    --add-data "mediapipe/modules/hand_landmark;." ^
+    --add-data "mediapipe/modules/hand_landmark;." ^
+    --add-data "mediapipe/modules/palm_detection;." ^
+    --add-data "mediapipe/modules/palm_detection;."
 
 echo "Deleting opencv dll"
 del dist\facetracker\cv2\opencv_videoio_*