Merge pull request #482 from luxonis/ci_stability_stress_test

CI stability stress test

Author: themarpe

.github/workflows/stability.workflow.yml

@@ -0,0 +1,54 @@
+name: DepthAI Core HIL Stability
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - develop
+    tags:
+      - 'v*'
+  pull_request:
+    branches:
+      - main
+      - develop
+
+# Only allow latest run on same branch to be tested
+concurrency:
+  group: ci-stability-${{ github.ref }}-1
+  cancel-in-progress: true
+
+jobs:
+
+  # Testing
+  test:
+    runs-on: ['self-hosted', 'hil-stability', 'linux']
+
+    steps:
+    - name: Cache .hunter folder
+      uses: actions/cache@v2
+      with:
+        path: $HOME/.hun_vanilla
+        key: hunter-linux-stability-vanilla
+    - uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+
+    # TODO also modify above hunter key to 'asan'
+    # - name: Specify ASAN toolchain path
+    #   run: echo "CMAKE_TOOLCHAIN_PATH=$PWD/cmake/toolchain/asan.cmake" >> $GITHUB_ENV
+
+    # - name: Configure, Build and Test
+    #   run: |
+    #     cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun_vanilla -D DEPTHAI_BUILD_TESTS=ON
+    #     cmake --build build --parallel 8 --config Release --target stability_stress_test
+    #     cd build
+    #     ../ci/stability_stress_test_combined.sh
+
+    # Release build
+    - name: Configure, Build and Test
+      run: |
+        cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun_vanilla -D DEPTHAI_BUILD_TESTS=ON
+        cmake --build build --parallel 8 --config Release --target stability_stress_test
+        cd build
+        ../ci/stability_stress_test_combined.sh 86400

ci/stability_stress_test_combined.sh

@@ -0,0 +1,38 @@
+# Cleanup in case the script dies
+cleanup() {
+    # kill all processes whose parent is this process
+    pkill -P $$
+}
+
+for sig in INT QUIT HUP TERM; do
+  trap "
+    cleanup
+    trap - $sig EXIT
+    kill -s $sig "'"$$"' "$sig"
+done
+trap cleanup EXIT
+
+# Print timeout if set, otherwise use default
+echo "Timeout set to: $1"
+
+# Run USB & PoE stability stress test
+DEPTHAI_PROTOCOL=usb timeout $(($1+30)) ./tests/stability_stress_test $1 &
+jobUsb=$!
+DEPTHAI_PROTOCOL=tcpip timeout $(($1+30)) ./tests/stability_stress_test $1 &
+jobTcpip=$!
+
+# Wait for tests and save result code
+wait $jobUsb ; resultUsb=$?
+wait $jobTcpip ; resultTcpip=$?
+
+# Print results
+echo "Stability test USB: $resultUsb"
+echo "Stability test PoE: $resultTcpip"
+
+# If both tests concluded successfully, exit with code 0
+if [[ "$resultUsb" == "0"  ]] && [[ "$resultTcpip" == "0" ]]; then
+    echo "Success!"
+    exit 0
+fi
+echo "Failed!"
+exit 1

tests/CMakeLists.txt

@@ -184,6 +184,13 @@ hunter_private_data(
     FILE "text-image-super-resolution-0001_2022.1.0_4shave.blob"
     LOCATION openvino_2022_1_blob
 )
+# YoloV4 resource
+hunter_private_data(
+    URL "https://artifacts.luxonis.com/artifactory/luxonis-depthai-data-local/network/yolo-v4-tiny-tf_openvino_2021.4_4shave.blob"
+    SHA1 "7da2f96f7300e3828940557e6a86ac6f243eef7e"
+    FILE "yolo-v4-tiny-tf_openvino_2021.4_4shave.blob"
+    LOCATION tiny_yolo_v4_2021-4_4shave_blob
+)
 
 
 # Add tests
@@ -234,4 +241,25 @@ dai_test_compile_definitions(filesystem_test_20 PRIVATE BLOB_PATH="${mobilenet_b
 dai_add_test(bootloader_version_test src/bootloader_version_test.cpp)
 
 # XLinkIn -> XLinkOut passthrough with large frames
-dai_add_test(xlink_roundtrip_test src/xlink_roundtrip_test.cpp)
+dai_add_test(xlink_roundtrip_test src/xlink_roundtrip_test.cpp)
+
+# Stability stress test (OAK-D oriented USB/PoE)
+# TODO
+# # dai_add_test(stability_stress_test src/stability_stress_test.cpp)
+option(DEPTHAI_STABILITY_STRESS_TEST_DEBUG "Enable visualization for stability stress test" OFF)
+add_executable(stability_stress_test src/stability_stress_test.cpp)
+add_default_flags(stability_stress_test LEAN)
+if(COMMAND target_clangformat_setup)
+    target_clangformat_setup(stability_stress_test "")
+endif()
+if(DEPTHAI_STABILITY_TEST_DEBUG)
+    target_link_libraries(stability_stress_test PRIVATE depthai::opencv opencv_highgui)
+    target_compile_definitions(stability_stress_test PRIVATE DEPTHAI_STABILITY_TEST_DEBUG)
+else()
+    target_link_libraries(stability_stress_test PRIVATE depthai::core)
+endif()
+target_compile_definitions(stability_stress_test PRIVATE BLOB_PATH="${tiny_yolo_v4_2021-4_4shave_blob}")
+# add_test(stability_stress_test stability_stress_test)
+# add_test(stability_stress_test_poe stability_stress_test)
+# set_tests_properties(stability_stress_test PROPERTIES ENVIRONMENT "UBSAN_OPTIONS=halt_on_error=1;DEPTHAI_PROTOCOL=usb" LABELS stability_usb)
+# set_tests_properties(stability_stress_test_poe PROPERTIES ENVIRONMENT "UBSAN_OPTIONS=halt_on_error=1;DEPTHAI_PROTOCOL=poe" LABELS stability_poe)