gha: add subsets to build

speed up build with subsets

Signed-off-by: Robert Gałat <[email protected]>

Author: RobertGalatNordic

.github/workflows/on-pr.yml

@@ -28,6 +28,6 @@ jobs:
     name: Build samples
     uses: ./.github/workflows/samples_build.yml
 
-  build_and_run_tests:
-    name: Build and run unit tests
-    uses: ./.github/workflows/run_tests.yml
+# build_and_run_tests:
+#   name: Build and run unit tests
+#   uses: ./.github/workflows/run_tests.yml

.github/workflows/samples_build.yml

@@ -2,6 +2,10 @@ name: Build Sidewalk samples
 on:
   workflow_call:
     inputs:
+      jobs_per_subset:
+        type: number
+        required: false
+        default: 3
       save_images:
         type: boolean
         required: false
@@ -16,6 +20,8 @@ jobs:
       options: --cpus 2 --privileged
     outputs:
       total_platforms_json: ${{ steps.platforms.outputs.total_platforms_json }}
+      subset_matrix: ${{ steps.subsets.outputs.matrix }}
+      subset_config: ${{ steps.subsets.outputs.config }}
     defaults:
       run:
         shell: nrfutil toolchain-manager launch --install-dir /root/ncs bash -- {0}
@@ -45,17 +51,46 @@ jobs:
           total_platforms_json=$(printf '%s' "${total_platforms[@]}" | jq -R . | jq -s -c .)
           echo "total_platforms_json=$total_platforms_json" >> $GITHUB_OUTPUT;
 
+      - name: subsets configuration
+        id: subsets
+        run: |
+          rm -rf twister-out*
+          west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="samples" --dry-run
+          total_platforms=$(jq -r '.testsuites |map({platform: .platform}) | unique | .[].platform' twister-out/testplan.json)
+          echo "[]" > configuration
+          for platform in $total_platforms; do
+            west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="samples" --dry-run --platform $platform
+            total_jobs=$(jq '.testsuites | length' twister-out/testplan.json)
+            subsets=$(( ( total_jobs + ${{inputs.jobs_per_subset}}) / ${{inputs.jobs_per_subset}}))
+            echo $subsets >> max
+            cat configuration | jq -c --arg platform_name $platform --arg subsets $subsets '. += [{"platform": $platform_name, "subsets": $subsets}]' > configuration_new
+            mv configuration_new configuration
+          done
+          sort max > max_sorted
+          max_subset=$(tail -n 1 max_sorted)
+          subset_matrix="[$(seq -s ', ' 1 $max_subset)]"
+          cat configuration
+          echo $subset_matrix
+
+          echo "matrix=$subset_matrix" >> $GITHUB_OUTPUT;
+          echo "config=`cat configuration`" >> $GITHUB_OUTPUT;
+
       - name: clean
         if: always()
         run: |
-          rm -rf twister*
+          rm -rf twister* sidewalk*
 
   build_samples:
     needs: [discover_targets]
     strategy:
       fail-fast: false
       matrix:
+        subset: ${{ fromJson(needs.discover_targets.outputs.subset_matrix) }}
         target: ${{ fromJson(needs.discover_targets.outputs.total_platforms_json) }}
+
     uses: ./.github/workflows/samples_build_target.yml
     with:
       target: ${{matrix.target}}
+      subset: ${{matrix.subset}}
+      subset_config: '${{needs.discover_targets.outputs.subset_config}}'
+# subset_config: '[{"platform":"nrf52840dk/nrf52840","subsets":"4"},{"platform":"nrf5340dk/nrf5340/cpuapp","subsets":"4"},{"platform":"nrf54l15dk/nrf54l10/cpuapp","subsets":"4"},{"platform":"nrf54l15dk/nrf54l15/cpuapp","subsets":"4"},{"platform":"nrf54l15dk/nrf54l15/cpuapp/ns","subsets":"4"},{"platform":"thingy53/nrf5340/cpuapp","subsets":"1"}]'

.github/workflows/samples_build_target.yml

@@ -9,10 +9,33 @@ on:
       target:
         type: string
         required: true
+      subset_config:
+        type: string
+        required: true
+      subset:
+        type: number
+        required: true
 
 jobs:
+  parse_config:
+    runs-on: ubuntu-24.04
+    name: ${{inputs.subset}} - ${{inputs.target}} config
+    outputs:
+      max_subset: ${{steps.subsets.outputs.max_subset}}
+    steps:
+      - name: subsets
+        id: subsets
+        run: |
+          echo '${{inputs.subset_config}}' > configuration.json
+          cat configuration.json
+          max_subsets=$(jq --arg target_name "${{inputs.target}}" '.[] | select(.platform == $target_name).subsets | tonumber' configuration.json)
+          echo "max_subset=$max_subsets"
+          echo "max_subset=$max_subsets" >> $GITHUB_OUTPUT
+
   build_samples:
-    name: ${{inputs.target}}
+    needs: [parse_config]
+    name: ${{inputs.subset}} - ${{inputs.target}}
+    if: ${{ inputs.subset <= needs.parse_config.outputs.max_subset }}
     runs-on: ubuntu-24.04
     container:
       image: ghcr.io/nrfconnect/sdk-sidewalk:main
@@ -57,23 +80,22 @@ jobs:
       - name: Twister build samples
         run: |
           source zephyr/zephyr-env.sh
-          west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="samples" --platform ${{ inputs.target}} --inline-logs --overflow-as-errors --enable-size-report --show-footprint -vvv --build-only
+          west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="samples" --platform ${{ inputs.target}} --inline-logs --overflow-as-errors --enable-size-report --show-footprint -vvv --build-only --subset ${{inputs.subset}}/${{needs.parse_config.outputs.max_subset}}
 
       - name: Print ccache stats
         run: |
           ccache -ps;
 
       - name: Prepare artifacts for upload
         run: |
-          touch dummy
+          mkdir subset_${{inputs.subset}}; cp twister-out/twister.json subset_${{inputs.subset}}
 
       - name: Upload build metadata
         uses: actions/upload-artifact@v4
         with:
-          name: sample-artifacts-meta-${{ steps.replace.outputs.target_board}}
+          name: sample-artifacts-meta-${{ steps.replace.outputs.target_board}}-${{inputs.subset}}
           path: |
-            dummy
-            twister-out/twister.json
+            subset_${{inputs.subset}}
             twister-out/**/build.log
             twister-out/**/.config
             twister-out/**/runners.yaml
@@ -84,9 +106,9 @@ jobs:
         uses: actions/upload-artifact@v4
         if: ${{ inputs.save_images }}
         with:
-          name: sample-artifacts-build-${{ steps.replace.outputs.target_board}}
+          name: sample-artifacts-build-${{ steps.replace.outputs.target_board}}-${{inputs.subset}}
           path: |
-            dummy
+            subset_${{inputs.subset}}
             twister-out/**/zephyr.elf
             twister-out/**/zephyr.hex
             twister-out/**/merged.hex