feat(use ncnn for model inference): add tooling and code for ncnn model inference with webassembly

Author: kethan1

.clang-format

@@ -0,0 +1,6 @@
+---
+BasedOnStyle: Google
+IndentWidth: 4
+ColumnLimit: 0
+---
+IndentExternBlock: Indent

.gitignore

@@ -1,5 +1,8 @@
 data/
-weights/
+models/
+emsdk/
+ncnn-20220701-webassembly/
+wasm_modules/
 
 ### Node ###
 # Logs

.parcelrc

@@ -1,4 +1,5 @@
 {
   "extends": "@parcel/config-default",
-  "namers": [ "parcel-namer-rewrite" ]
+  "namers": ["parcel-namer-rewrite"],
+  "reporters":  ["...", "parcel-reporter-static-files-copy"]
 }

CMakeLists.txt

@@ -0,0 +1,34 @@
+project(ncnn-webassembly-yolov5)
+
+cmake_minimum_required(VERSION 3.10)
+
+set(CMAKE_BUILD_TYPE release)
+
+if(NOT WASM_FEATURE)
+    message(FATAL_ERROR "You must pass cmake option -DWASM_FEATURE and possible values are basic, simd, threads and simd-threads")
+endif()
+
+set(ncnn_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ncnn-20220701-webassembly/${WASM_FEATURE}/lib/cmake/ncnn")
+find_package(ncnn REQUIRED)
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -sFORCE_FILESYSTEM=1 -sINITIAL_MEMORY=512MB")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -sFORCE_FILESYSTEM=1 -sINITIAL_MEMORY=512MB")
+set(CMAKE_EXECUTBLE_LINKER_FLAGS "${CMAKE_EXECUTBLE_LINKER_FLAGS} -sFORCE_FILESYSTEM=1 -sINITIAL_MEMORY=512MB")
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -sEXPORTED_FUNCTIONS='[\"_yolov5_ncnn\",\"_yolov5_ncnn_inference\"]' -sEXPORTED_RUNTIME_METHODS='[\"ccall\",\"cwrap\",\"getValue\",\"setValue\"]' --preload-file '${CMAKE_CURRENT_SOURCE_DIR}/models@.' -sLLD_REPORT_UNDEFINED -sERROR_ON_UNDEFINED_SYMBOLS=0 -sASSERTIONS")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -sEXPORTED_FUNCTIONS='[\"_yolov5_ncnn\",\"_yolov5_ncnn_inference\"]' -sEXPORTED_RUNTIME_METHODS='[\"ccall\",\"cwrap\",\"getValue\",\"setValue\"]' --preload-file '${CMAKE_CURRENT_SOURCE_DIR}/models@.' -sLLD_REPORT_UNDEFINED -sERROR_ON_UNDEFINED_SYMBOLS=0 -sASSERTIONS")
+set(CMAKE_EXECUTBLE_LINKER_FLAGS "${CMAKE_EXECUTBLE_LINKER_FLAGS} -sEXPORTED_FUNCTIONS='[\"_yolov5_ncnn\",\"_yolov5_ncnn_inference\"]' -sEXPORTED_RUNTIME_METHODS='[\"ccall\",\"cwrap\",\"getValue\",\"setValue\"]' --preload-file '${CMAKE_CURRENT_SOURCE_DIR}/models@.' -sLLD_REPORT_UNDEFINED -sERROR_ON_UNDEFINED_SYMBOLS=0 -sASSERTIONS")
+
+if(${WASM_FEATURE} MATCHES "threads")
+    MESSAGE(STATUS "running threads")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fopenmp -pthread -s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=4")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -pthread -s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=4")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fopenmp -pthread -s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=4")
+endif()
+
+MESSAGE(STATUS "building")
+
+add_executable(yolov5-${WASM_FEATURE} src/yolov5.cpp src/yolov5ncnn.cpp src/globals.cpp)
+MESSAGE(STATUS "linking")
+target_link_libraries(yolov5-${WASM_FEATURE} ncnn)
+MESSAGE(STATUS "copying files to output directory")

app.py

@@ -1,37 +1,13 @@
-from flask import Flask, render_template, request
-from PIL import Image
-from model_abstractions import YOLOv5_Model
-from parabola_manipulation import calculate_parabola_parameters
+from flask import Flask, render_template
+
 
 app = Flask(__name__, template_folder="dist/templates", static_folder="dist/static")
-ball_detection_model = YOLOv5_Model("weights/ball_detection_model.pt")
 
 
 @app.route("/")
 def home():
     return render_template("home.html")
 
 
-@app.route("/analyze_gameplay")
-def analyze_gameplay():
-    return render_template("analyze_gameplay.html")
-
-
-@app.route("/api/v1/detect-ball", methods=["POST"])
-def detect_ball():
-    image = Image.open(request.files["file"])
-    if predictions := ball_detection_model.predict_coords(image):
-        return max(predictions, key=lambda x: x["confidence"])
-    return {}
-
-
-@app.route("/api/v1/calculate_parabola", methods=["POST"])
-def calculate_parabola():
-    # structure: {'point1': {'x': 1, 'y': 2}, 'point2': {'x': 3, 'y': 4}, 'point3': {'x': 5, 'y': 6}}
-    (x1, y1), (x2, y2), (x3, y3) = map(dict.values, request.json.values())
-    A, B, C = calculate_parabola_parameters(x1, y1, x2, y2, x3, y3)
-    return {"A": A, "B": B, "C": C}
-
-
 if __name__ == "__main__":
     app.run(debug=True)

os_switcher.py

@@ -0,0 +1,44 @@
+import re
+import os
+import argparse
+import platform
+import subprocess
+
+
+parser = argparse.ArgumentParser(description="Run different commands based on the OS")
+parser.add_argument("--windows", help="Command to run on Windows", required=True)
+parser.add_argument("--linux", help="Command to run on Linux", required=True)
+parser.add_argument("--macos", help="Command to run on MacOS, defaults to Linux command")
+args = parser.parse_args()
+
+if args.macos is None:
+    args.macos = args.linux
+
+
+def preprocess(string):
+    split = re.findall(r"\{(.*?)\}", string)
+    final_sub = {}
+    for sub in split:
+        sub_split = sub.split(":")
+        if sub_split[0] == "env":
+            final_sub[sub] = os.environ[sub_split[1]]
+    for old, new in final_sub.items():
+        string = string.replace("{" + old + "}", new)
+    return string
+
+
+args.windows = preprocess(args.windows)
+args.linux = preprocess(args.linux)
+args.macos = preprocess(args.macos)
+
+if platform.system() == "Windows":
+    popen_kwargs = {"stdin": subprocess.PIPE}
+    proc = subprocess.Popen(args.windows, **popen_kwargs, shell=True)
+elif platform.system() == "Linux":
+    popen_kwargs = {"stdin": subprocess.PIPE}
+    proc = subprocess.Popen(args.linux, **popen_kwargs, shell=True)
+elif platform.system() == "Darwin":
+    popen_kwargs = {"stdin": subprocess.PIPE}
+    proc = subprocess.Popen(args.macos, **popen_kwargs, shell=True)
+
+proc.wait()

package-lock.json

@@ -6,6 +6,7 @@
     "events": "^3.3.0",
     "parcel": "^2.6.2",
     "parcel-namer-rewrite": "^2.0.0-rc.2",
+    "parcel-reporter-static-files-copy": "^1.3.4",
     "path-browserify": "^1.0.1",
     "postcss": "^8.4.14",
     "process": "^0.11.10",
@@ -17,5 +18,9 @@
       "(.*)\\.(svg|png|gif|jpg|css|js)": "static/$1.{hash}$2",
       "(.*)\\.(html)": "templates/$1.$2"
     }
+  },
+  "staticFiles": {
+    "staticPath": "static/wasm_modules",
+    "staticOutPath": "static/wasm_modules"
   }
 }

package.json

@@ -22,21 +22,42 @@ scipy = "^1.8.1"
 torchvision = "^0.13.0"
 protobuf = "<4.21.3"
 seaborn = "^0.11.2"
+imagesize = "^1.4.1"
 
 [tool.poetry.dev-dependencies]
 tqdm = "^4.64.0"
 
 [tool.poe.tasks.watch]
 shell = "npx parcel watch templates/*.html --no-source-maps --dist-dir dist"
-interpreter = ["pwsh", "posix"]
+interpreter = ["powershell", "posix"]
 
 [tool.poe.tasks.build]
 shell = "npx parcel build templates/*.html --no-source-maps --dist-dir dist"
-interpreter = ["pwsh", "posix"]
+interpreter = ["powershell", "posix"]
 
 [tool.poe.tasks.server]
 shell = "poetry run python app.py"
-interpreter = ["pwsh", "posix"]
+interpreter = ["powershell", "posix"]
+
+[tool.poe.tasks.build_basic_wasm]
+shell = '''python os_switcher.py --windows 'mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=\"{env:EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake\" -DWASM_FEATURE=basic -G \"Unix Makefiles\" .. && cmake --build . --config Release -j 4 && move yolov5* ../static/wasm_modules && cd .. && del /s /q build > NUL && rmdir /s /q build' --linux 'mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=\"{env:EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake\" -DWASM_FEATURE=basic -G \"Unix Makefiles\" .. && cmake --build . --config Release -j 4 && mv yolov5* ../static/wasm_modules && cd .. && rm -r build''''
+interpreter = ["powershell", "posix"]
+
+[tool.poe.tasks.build_simd_wasm]
+shell = '''python os_switcher.py --windows 'mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=\"{env:EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake\" -DWASM_FEATURE=simd -G \"Unix Makefiles\" .. && cmake --build . --config Release -j 4 && move yolov5* ../static/wasm_modules && cd .. && del /s /q build > NUL && rmdir /s /q build' --linux 'mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=\"{env:EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake\" -DWASM_FEATURE=simd -G \"Unix Makefiles\" .. && cmake --build . --config Release -j 4 && mv yolov5* ../static/wasm_modules && cd .. && rm -r build''''
+interpreter = ["powershell", "posix"]
+
+[tool.poe.tasks.build_threads_wasm]
+shell = '''python os_switcher.py --windows 'mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=\"{env:EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake\" -DWASM_FEATURE=threads -G \"Unix Makefiles\" .. && cmake --build . --config Release -j 4 && move yolov5* ../static/wasm_modules && cd .. && del /s /q build > NUL && rmdir /s /q build' --linux 'mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=\"{env:EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake\" -DWASM_FEATURE=threads -G \"Unix Makefiles\" .. && cmake --build . --config Release -j 4 && mv yolov5* ../static/wasm_modules && cd .. && rm -r build''''
+interpreter = ["powershell", "posix"]
+
+[tool.poe.tasks.build_simd_threads_wasm]
+shell = '''python os_switcher.py --windows 'mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=\"{env:EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake\" -DWASM_FEATURE=simd-threads -G \"Unix Makefiles\" .. && cmake --build . --config Release -j 4 && move yolov5* ../static/wasm_modules && cd .. && del /s /q build > NUL && rmdir /s /q build' --linux 'mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=\"{env:EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake\" -DWASM_FEATURE=simd-threads -G \"Unix Makefiles\" .. && cmake --build . --config Release -j 4 && mv yolov5* ../static/wasm_modules && cd .. && rm -r build''''
+interpreter = ["powershell", "posix"]
+
+[tool.poe.tasks.build_wasm]
+shell = "poetry poe build_basic && poetry poe build_simd && poetry poe build_threads && poetry poe build_simd_threads"
+interpreter = ["powershell", "posix"]
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]