west: add west.yml

add west.yml to sidewalk project
update github actions
improve build and test in GHA,
by dynamically selecting number of runners

nrfconnect/sdk-nrf#20901

update template for PRs

Signed-off-by: Robert Gałat <[email protected]>

Author: RobertGalatNordic

.github/pull_request_template.md

@@ -1,12 +1,6 @@
 ## CI parameters
 
 ```yaml
-Github_actions:
-  #(branch, hash, pull/XXX/head)
-  NRF_revision: main
-
-  # Do not change after creating PR
-  Create_NRF_PR: false
 Jenkins:
   test-sdk-sidewalk: master
   # To reconfigure functional tests:

.github/workflows/on-commit.yml

@@ -33,37 +33,13 @@ jobs:
     with:
       reference_commit: "HEAD~${{needs.get_base_commit.outputs.commit_number}}"
 
-  discover_target_boards:
-    runs-on: ubuntu-24.04
-    outputs:
-      targets: ${{ steps.platforms.outputs.targets }}
-    steps:
-      - name: install yq
-        run: pip install yq
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          path: sidewalk
-
-      - name: get_platforms
-        id: platforms
-        run: |
-          result=$(yq '.common.platform_allow' sidewalk/samples/sid_end_device/sample.yaml -c)
-          echo "targets=$result" >> $GITHUB_OUTPUT;
-          echo "$result"
-
   build_samples:
-    needs: [discover_target_boards]
+    name: Build samples
     uses: ./.github/workflows/samples_build.yml
-    with:
-      boards_to_test: ${{needs.discover_target_boards.outputs.targets}}
 
   build_and_run_tests:
-    needs: [discover_target_boards]
+    name: Build and run unit tests
     uses: ./.github/workflows/run_tests.yml
-    with:
-      boards_to_test: ${{needs.discover_target_boards.outputs.targets}}
 
   Post_fail_to_Teams:
     needs: [verify_changes, build_samples, build_and_run_tests]

.github/workflows/on-pr.yml

@@ -3,43 +3,6 @@ on:
   pull_request:
 
 jobs:
-  Run_Configuration:
-    runs-on: ubuntu-24.04
-    outputs:
-      NRF_revision: ${{ steps.config.outputs.NRF_revision }}
-    steps:
-      - name: Read body of PR
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /repos/${{github.repository}}/pulls/${{github.event.number}} --jq '.body' > pr_body.md
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.9'
-
-      - name: install yq
-        run: pip install yq
-
-      - name: Get configuration
-        id: config
-        shell: bash {0}
-        run: |
-          csplit pr_body.md /\`\`\`yaml/1
-          csplit xx01 /\`\`\`/
-          mv xx00 configuration.yaml
-          cat configuration.yaml
-          revision=$(yq '.Github_actions.NRF_revision' configuration.yaml)
-          if [ $? != 0 ]; then
-            echo "Failed to read configuration use default main for NRF_revision"
-            echo "NRF_revision=main" >> $GITHUB_OUTPUT;
-          else
-            echo "NRF_revision=$revision" >> $GITHUB_OUTPUT;
-          fi;
-
-      - name: print nrf revision
-        run: |
-          echo ${{ steps.config.outputs.NRF_revision }}
 
   verify_PR:
     uses: ./.github/workflows/validate_code_style.yml
@@ -61,36 +24,10 @@ jobs:
             PR_number
           include-hidden-files: true
 
-  discover_target_boards:
-    runs-on: ubuntu-24.04
-    outputs:
-      targets: ${{ steps.platforms.outputs.targets }}
-    steps:
-      - name: install yq
-        run: pip install yq
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          path: sidewalk
-
-      - name: get_platforms
-        id: platforms
-        run: |
-          result=$(yq '.common.platform_allow' sidewalk/samples/sid_end_device/sample.yaml -c)
-          echo "targets=$result" >> $GITHUB_OUTPUT;
-          echo "$result"
-
   build_samples:
-    needs: [Run_Configuration, discover_target_boards]
+    name: Build samples
     uses: ./.github/workflows/samples_build.yml
-    with:
-      nrf_revision: ${{needs.Run_Configuration.outputs.NRF_revision}}
-      boards_to_test: ${{needs.discover_target_boards.outputs.targets}}
 
   build_and_run_tests:
-    needs: [Run_Configuration, discover_target_boards]
+    name: Build and run unit tests
     uses: ./.github/workflows/run_tests.yml
-    with:
-      nrf_revision: ${{needs.Run_Configuration.outputs.NRF_revision}}
-      boards_to_test: ${{needs.discover_target_boards.outputs.targets}}

.github/workflows/on-pr_nrf_manifest_update_PR.yml

@@ -53,10 +53,7 @@ jobs:
       - name: Get revision of nrf to use
         id: revision
         run: |
-          REF_NAME="${{ github.ref_name }}"
-          if [ -z "$REF_NAME" ]; then
-            REF_NAME="main"  # Set your default value here
-          fi
+          REF_NAME=$(yq '.manifest.projects | map(select(.name == "nrf"))[0].revision' west.yml)
           echo "REF_NAME=$REF_NAME" >> $GITHUB_OUTPUT
           echo "Ref name: $REF_NAME"
 

.github/workflows/on_docker_change.yml

@@ -2,17 +2,71 @@ name: Build Sidewalk samples
 on:
   workflow_call:
     inputs:
-      nrf_revision:
-        type: string
+      jobs_per_subset:
+        type: number
         required: false
-        default: "main"
-      boards_to_test:
-        type: string
-        required: true
-        default: '["nrf52840dk/nrf52840", "nrf5340dk/nrf5340/cpuapp", "nrf54l15dk/nrf54l15/cpuapp", "nrf54l15dk/nrf54l15/cpuapp/ns", "nrf54l15dk/nrf54l10/cpuapp"]'
+        default: 6
 
 jobs:
+  discover_targets:
+    runs-on: self-hosted
+    container:
+      image: ghcr.io/nrfconnect/sdk-sidewalk:main
+      options: --cpus 2 --privileged
+    outputs:
+      total_platforms_json: ${{ steps.subsets.outputs.total_platforms_json }}
+      subset_matrix: ${{ steps.subsets.outputs.matrix }}
+      subset_config: ${{ steps.subsets.outputs.config }}
+    defaults:
+      run:
+        shell: nrfutil toolchain-manager launch --install-dir /root/ncs bash -- {0}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          path: sidewalk
+
+      - name: update NRF
+        run: |
+          rm -rf .west;
+          west init -l sidewalk --mf west.yml &&
+          west update -n -o=--depth=1 --path-cache /workdir/
+
+      - name: subsets configuration
+        id: subsets
+        run: |
+          rm -rf twister-out* max max_sorted configuration
+          west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="Github_tests"  --exclude-platform native_sim --exclude-platform native_posix --exclude-platform unit_testing  --dry-run
+          total_platforms=$(jq -r '.testsuites |map({platform: .platform}) | unique | .[].platform' twister-out/testplan.json)
+          total_platforms_json=$(printf '%s' "${total_platforms[@]}" | jq -R . | jq -s -c .)
+          echo "{}" > configuration
+          for platform in $total_platforms; do
+            west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="Github_tests" --dry-run --platform $platform
+            total_jobs=$(jq '.testsuites | length' twister-out/testplan.json)
+            subsets=$(( ( total_jobs + ${{inputs.jobs_per_subset}}) / ${{inputs.jobs_per_subset}}))
+            echo $subsets >> max
+            cat configuration | jq -c --arg platform_name $platform --arg subsets $subsets '. += {($platform_name): $subsets|tonumber }' > configuration_new
+            mv configuration_new configuration
+          done
+          sort max > max_sorted
+          max_subset=$(tail -n 1 max_sorted)
+          subset_matrix="[$(seq -s ', ' 1 $max_subset)]"
+          cat configuration
+          echo $subset_matrix
+
+          echo "total_platforms_json=$total_platforms_json" >> $GITHUB_OUTPUT;
+          echo "matrix=$subset_matrix" >> $GITHUB_OUTPUT;
+          echo "config=`cat configuration`" >> $GITHUB_OUTPUT;
+
+      - name: clean
+        if: always()
+        run: |
+          rm -rf twister*
+
   build_and_run_x86_tests:
+    name: Run UT on x86
     runs-on: ubuntu-24.04
     container:
       image: ghcr.io/nrfconnect/sdk-sidewalk:main
@@ -30,16 +84,9 @@ jobs:
 
       - name: update NRF
         run: |
-          cd sidewalk &&  git rev-parse HEAD > /workdir/sidewalk_hash && cd .. &&
-          cp -r /workdir/nrf nrf;
-          west init -l nrf &&
-          cd nrf &&
-          git fetch origin ${{inputs.nrf_revision}}:PR-branch &&
-          git checkout PR-branch &&
-          cd .. &&
-          west config manifest.group-filter +sidewalk &&
-          west update -n -o=--depth=1 --path-cache /workdir/ &&
-          cd sidewalk && git checkout `cat /workdir/sidewalk_hash` && rm -rf /workdir/sidewalk_hash
+          rm -rf .west;
+          west init -l sidewalk --mf west.yml &&
+          west update -n -o=--depth=1 --path-cache /workdir/
 
       - name: Install Python dependencies
         run: |
@@ -84,18 +131,18 @@ jobs:
 
 
   build-and-test_on_hw:
+    needs: [discover_targets]
     strategy:
       fail-fast: false
       matrix:
-        subset: [1, 2]
-        board: ${{ fromJson(inputs.boards_to_test) }}
+        subset: ${{ fromJson(needs.discover_targets.outputs.subset_matrix) }}
+        target: ${{ fromJson(needs.discover_targets.outputs.total_platforms_json) }}
     uses: ./.github/workflows/run_tests_dut.yml
     with:
-      subset: ${{ matrix.subset }}
-      max_subsets: 2
       run_tests: true
-      target_board: ${{ matrix.board }}
-      nrf_revision: ${{ inputs.nrf_revision }}
+      target: ${{matrix.target}}
+      subset: ${{matrix.subset}}
+      subset_config: '${{needs.discover_targets.outputs.subset_config}}'
 
   merge_tests_on_hw:
     needs: build-and-test_on_hw

.github/workflows/run_tests.yml

@@ -13,7 +13,7 @@ on:
 
 jobs:
   run_dut_ut:
-    name: "Run DUT tests  (${{inputs.target_board}} - ${{ inputs.subset }})"
+    name: "Run DUT tests  (${{inputs.target_board}})"
     runs-on: [self-hosted, linux, x64, "${{inputs.target_board}}"]
     container:
       image: ghcr.io/nrfconnect/sdk-sidewalk:main
@@ -27,24 +27,25 @@ jobs:
         shell: nrfutil toolchain-manager launch --install-dir /root/ncs bash -- {0}
 
     steps:
+      - name: clean artifacts
+        if: always()
+        run: |
+          rm -rf pytest-sidewalk
+          rm -rf twister-out*
+          rm -rf sidewalk
+          rm -rf .west
+
       - name: Checkout
         uses: actions/checkout@v4
         with:
+          fetch-depth: 0
           path: sidewalk
 
       - name: update NRF
         run: |
-          rm -rf .west nrf;
-          cd sidewalk &&  git rev-parse HEAD > /workdir/sidewalk_hash && cd ..
-          west init -m https://github.com/nrfconnect/sdk-nrf --mr main . &&
-          cd nrf &&
-          git fetch origin ${{inputs.nrf_revision}}:branch_to_run &&
-          git checkout branch_to_run &&
-          git rev-parse HEAD &&
-          cd .. &&
-          west config manifest.group-filter +sidewalk &&
-          west update -n -o=--depth=1 --path-cache /workdir/ &&
-          cd sidewalk && git checkout `cat /workdir/sidewalk_hash` && rm -rf /workdir/sidewalk_hash
+          rm -rf .west;
+          west init -l sidewalk --mf west.yml &&
+          west update -n -o=--depth=1 --path-cache /workdir/
 
       - name: Replace slash with underscore
         id: replace
@@ -102,5 +103,7 @@ jobs:
         if: always()
         run: |
           rm -rf pytest-sidewalk
-          cd nrf; git checkout main; git branch -D branch_to_run;
-          rm -rf PACKAGE_ARTIFACTS_* twister-out
+          rm -rf twister-out*
+          rm -rf sidewalk_tmp
+          rm -rf sidewalk
+          rm -rf .west