gha: use subsets for tests

RobertGalatNordic · RobertGalatNordic · commit 68446e83fea0 · 2025-03-12T12:59:24.000Z
paralelize build and run of UT on boards

Signed-off-by: Robert Gałat &lt;robert.galat@nordicsemi.no&gt;
diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml
@@ -28,6 +28,6 @@ jobs:
     name: Build samples
     uses: ./.github/workflows/samples_build.yml
 
-# build_and_run_tests:
-#   name: Build and run unit tests
-#   uses: ./.github/workflows/run_tests.yml
+  build_and_run_tests:
+    name: Build and run unit tests
+    uses: ./.github/workflows/run_tests.yml
diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
@@ -1,6 +1,11 @@
 name: Build Sidewalk samples
 on:
   workflow_call:
+    inputs:
+      jobs_per_subset:
+        type: number
+        required: false
+        default: 3
 
 jobs:
   discover_targets:
@@ -10,6 +15,8 @@ jobs:
       options: --cpus 2 --privileged
     outputs:
       total_platforms_json: ${{ steps.platforms.outputs.total_platforms_json }}
+      subset_matrix: ${{ steps.subsets.outputs.matrix }}
+      subset_config: ${{ steps.subsets.outputs.config }}
     defaults:
       run:
         shell: nrfutil toolchain-manager launch --install-dir /root/ncs bash -- {0}
@@ -39,6 +46,30 @@ jobs:
           total_platforms_json=$(printf '%s' "${total_platforms[@]}" | jq -R . | jq -s -c .)
           echo "total_platforms_json=$total_platforms_json" >> $GITHUB_OUTPUT;
 
+      - name: subsets configuration
+        id: subsets
+        run: |
+          rm -rf twister-out* max max_sorted configuration
+          west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="Github_tests"  --exclude-platform native_sim --exclude-platform native_posix --exclude-platform unit_testing  --dry-run
+          total_platforms=$(jq -r '.testsuites |map({platform: .platform}) | unique | .[].platform' twister-out/testplan.json)
+          echo "[]" > configuration
+          for platform in $total_platforms; do
+            west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="Github_tests" --dry-run --platform $platform
+            total_jobs=$(jq '.testsuites | length' twister-out/testplan.json)
+            subsets=$(( ( total_jobs + ${{inputs.jobs_per_subset}}) / ${{inputs.jobs_per_subset}}))
+            echo $subsets >> max
+            cat configuration | jq -c --arg platform_name $platform --arg subsets $subsets '. += [{"platform": $platform_name, "subsets": $subsets}]' > configuration_new
+            mv configuration_new configuration
+          done
+          sort max > max_sorted
+          max_subset=$(tail -n 1 max_sorted)
+          subset_matrix="[$(seq -s ', ' 1 $max_subset)]"
+          cat configuration
+          echo $subset_matrix
+
+          echo "matrix=$subset_matrix" >> $GITHUB_OUTPUT;
+          echo "config=`cat configuration`" >> $GITHUB_OUTPUT;
+
       - name: clean
         if: always()
         run: |
@@ -115,11 +146,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        board: ${{ fromJson(needs.discover_targets.outputs.total_platforms_json) }}
+        subset: ${{ fromJson(needs.discover_targets.outputs.subset_matrix) }}
+        target: ${{ fromJson(needs.discover_targets.outputs.total_platforms_json) }}
     uses: ./.github/workflows/run_tests_dut.yml
     with:
       run_tests: true
-      target_board: ${{ matrix.board }}
+      target: ${{matrix.target}}
+      subset: ${{matrix.subset}}
+      subset_config: '${{needs.discover_targets.outputs.subset_config}}'
 
   merge_tests_on_hw:
     needs: build-and-test_on_hw
diff --git a/.github/workflows/run_tests_dut.yml b/.github/workflows/run_tests_dut.yml
@@ -2,19 +2,41 @@ name: Build and run DUT tests
 on:
   workflow_call:
     inputs:
-      target_board:
-        description: "target board for tests"
+      target:
         type: string
         required: true
+      subset_config:
+        type: string
+        required: true
+      subset:
+        type: number
+        required: true
       run_tests:
         description: ""
         required: false
         type: boolean
         default: true
 
 jobs:
+  parse_config:
+    runs-on: ubuntu-24.04
+    name: ${{inputs.subset}} - ${{inputs.target}} config
+    outputs:
+      max_subset: ${{steps.subsets.outputs.max_subset}}
+    steps:
+      - name: subsets
+        id: subsets
+        run: |
+          echo '${{inputs.subset_config}}' > configuration.json
+          cat configuration.json
+          max_subsets=$(jq --arg target_name "${{inputs.target}}" '.[] | select(.platform == $target_name).subsets | tonumber' configuration.json)
+          echo "max_subset=$max_subsets"
+          echo "max_subset=$max_subsets" >> $GITHUB_OUTPUT
+
   build_dut_tests:
-    name: "Build UT for (${{inputs.target_board}})"
+    needs: [parse_config]
+    if: ${{ inputs.subset <= needs.parse_config.outputs.max_subset }}
+    name: "Build UT ${{inputs.subset}} - ${{inputs.target}}"
     runs-on: ubuntu-24.04
     container:
       image: ghcr.io/nrfconnect/sdk-sidewalk:main
@@ -44,7 +66,7 @@ jobs:
       - name: Replace slash with underscore
         id: replace
         run: |
-          TARGET_BOARD=${{ inputs.target_board }}
+          TARGET_BOARD=${{ inputs.target }}
           TARGET_BOARD=${TARGET_BOARD//\//_}
           echo "target_board=$TARGET_BOARD" >> $GITHUB_OUTPUT
 
@@ -59,7 +81,7 @@ jobs:
       - name: Build DUT test artifacts
         run: |
           source zephyr/zephyr-env.sh
-          west twister --test-config sidewalk/test_config.yaml --level="Github_tests" --platform ${{inputs.target_board}} --testsuite-root sidewalk  --inline-logs --overflow-as-errors -vvv --prep-artifacts-for-testing --package-artifacts PACKAGE_ARTIFACTS.tar.bz2
+          west twister --test-config sidewalk/test_config.yaml --level="Github_tests" --platform ${{inputs.target}} --testsuite-root sidewalk  --inline-logs --overflow-as-errors -vvv --prep-artifacts-for-testing --package-artifacts PACKAGE_ARTIFACTS_${{ inputs.subset }}.tar.bz2 --subset ${{ inputs.subset }}/${{needs.parse_config.outputs.max_subset}}
 
       - name: Print ccache stats
         run: |
@@ -68,16 +90,15 @@ jobs:
       - name: Save build artifacts
         uses: actions/upload-artifact@v4
         with:
-          path: PACKAGE_ARTIFACTS.tar.bz2
-          name: DUT_artifacts_${{steps.replace.outputs.target_board}}
+          path: PACKAGE_ARTIFACTS_${{ inputs.subset }}.tar.bz2
+          name: DUT_artifacts_${{steps.replace.outputs.target_board}}_${{ inputs.subset }}
           include-hidden-files: true
 
   run_dut_ut:
-    needs: [build_dut_tests]
-
-    if: ${{ fromJson(inputs.run_tests) }}
-    name: "Run on ${{inputs.target_board}}"
-    runs-on: [self-hosted, linux, x64, "${{inputs.target_board}}"]
+    needs: [build_dut_tests, parse_config]
+    if: ${{ fromJson(inputs.run_tests) }} and ${{ inputs.subset <= needs.parse_config.outputs.max_subset }}
+    name: "Run UT ${{ inputs.subset }} - ${{inputs.target}}"
+    runs-on: [self-hosted, linux, x64, "${{inputs.target}}"]
     container:
       image: ghcr.io/nrfconnect/sdk-sidewalk:main
       options: --cpus 2 --privileged
@@ -114,7 +135,7 @@ jobs:
       - name: Replace slash with underscore
         id: replace
         run: |
-          TARGET_BOARD=${{ inputs.target_board }}
+          TARGET_BOARD=${{ inputs.target }}
           TARGET_BOARD=${TARGET_BOARD//\//_}
           echo "target_board=$TARGET_BOARD" >> $GITHUB_OUTPUT
 
@@ -125,11 +146,11 @@ jobs:
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
-          name: DUT_artifacts_${{steps.replace.outputs.target_board}}
+          name: DUT_artifacts_${{steps.replace.outputs.target_board}}_${{ inputs.subset }}
 
       - name: Unwrap artifacts
         run: |
-          tar -xf PACKAGE_ARTIFACTS.tar.bz2
+          tar -xf PACKAGE_ARTIFACTS_${{ inputs.subset }}.tar.bz2
 
       - name: Run Tests
         run: |
@@ -138,13 +159,13 @@ jobs:
 
       - name: rename_twister_xml_report
         run: |
-          mv twister-out/twister.xml twister-out/${{steps.replace.outputs.target_board}}_twister.xml
+          mv twister-out/twister.xml twister-out/${{steps.replace.outputs.target_board}}_${{ inputs.subset }}_twister.xml
 
       - name: Upload test results
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: tests-dut_result_${{steps.replace.outputs.target_board}}
+          name: tests-dut_result_${{steps.replace.outputs.target_board}}_${{ inputs.subset }}
           path: |
             twister-out/${{steps.replace.outputs.target_board}}_twister.xml
             twister-out/**/handler.log
diff --git a/.github/workflows/samples_build.yml b/.github/workflows/samples_build.yml
@@ -54,7 +54,7 @@ jobs:
       - name: subsets configuration
         id: subsets
         run: |
-          rm -rf twister-out*
+          rm -rf twister-out* max max_sorted configuration
           west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="samples" --dry-run
           total_platforms=$(jq -r '.testsuites |map({platform: .platform}) | unique | .[].platform' twister-out/testplan.json)
           echo "[]" > configuration
@@ -93,4 +93,25 @@ jobs:
       target: ${{matrix.target}}
       subset: ${{matrix.subset}}
       subset_config: '${{needs.discover_targets.outputs.subset_config}}'
-# subset_config: '[{"platform":"nrf52840dk/nrf52840","subsets":"4"},{"platform":"nrf5340dk/nrf5340/cpuapp","subsets":"4"},{"platform":"nrf54l15dk/nrf54l10/cpuapp","subsets":"4"},{"platform":"nrf54l15dk/nrf54l15/cpuapp","subsets":"4"},{"platform":"nrf54l15dk/nrf54l15/cpuapp/ns","subsets":"4"},{"platform":"thingy53/nrf5340/cpuapp","subsets":"1"}]'
+
+  merge_sample_artifacts:
+    needs: build_samples
+    runs-on: ubuntu-24.04
+    permissions: write-all
+    steps:
+      - name: Merge Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          delete-merged: true
+          name: sample-artifacts-meta
+          pattern: sample-artifacts-meta-*
+          include-hidden-files: true
+
+      - name: Merge Artifacts
+        uses: actions/upload-artifact/merge@v4
+        if: ${{ inputs.save_images }}
+        with:
+          delete-merged: true
+          name: sample-artifacts
+          pattern: sample-artifacts-build-*
+          include-hidden-files: true
diff --git a/.github/workflows/samples_build_target.yml b/.github/workflows/samples_build_target.yml
@@ -88,14 +88,15 @@ jobs:
 
       - name: Prepare artifacts for upload
         run: |
-          mkdir subset_${{inputs.subset}}; cp twister-out/twister.json subset_${{inputs.subset}}
+          mkdir -p subsets/${{inputs.subset}}_of_${{ needs.parse_config.outputs.max_subset }}_${{ steps.replace.outputs.target_board}}
+          cp twister-out/twister.json subsets/${{inputs.subset}}_of_${{ needs.parse_config.outputs.max_subset }}_${{ steps.replace.outputs.target_board}}
 
       - name: Upload build metadata
         uses: actions/upload-artifact@v4
         with:
           name: sample-artifacts-meta-${{ steps.replace.outputs.target_board}}-${{inputs.subset}}
           path: |
-            subset_${{inputs.subset}}
+            subsets
             twister-out/**/build.log
             twister-out/**/.config
             twister-out/**/runners.yaml
@@ -108,7 +109,7 @@ jobs:
         with:
           name: sample-artifacts-build-${{ steps.replace.outputs.target_board}}-${{inputs.subset}}
           path: |
-            subset_${{inputs.subset}}
+            subsets
             twister-out/**/zephyr.elf
             twister-out/**/zephyr.hex
             twister-out/**/merged.hex
