GHA: add test_config and update GHA

prepare gitHub automation for unit test refactoring

Signed-off-by: Robert Gałat <[email protected]>

Author: RobertGalatNordic

.github/workflows/on-commit.yml

@@ -33,11 +33,37 @@ jobs:
     with:
       reference_commit: "HEAD~${{needs.get_base_commit.outputs.commit_number}}"
 
+  discover_target_boards:
+    runs-on: ubuntu-latest
+    outputs:
+      targets: ${{ steps.platforms.outputs.targets }}
+    steps:
+      - name: install yq
+        run: pip install yq
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          path: sidewalk
+
+      - name: get_platforms
+        id: platforms
+        run: |
+          result=$(yq '.common.platform_allow' sidewalk/samples/sid_end_device/sample.yaml -c)
+          echo "targets=$result" >> $GITHUB_OUTPUT;
+          echo "$result"
+
   build_samples:
+    needs: [discover_target_boards]
     uses: ./.github/workflows/samples_build.yml
+    with:
+      boards_to_test: ${{needs.discover_target_boards.outputs.targets}}
 
   build_and_run_tests:
+    needs: [discover_target_boards]
     uses: ./.github/workflows/run_tests.yml
+    with:
+      boards_to_test: ${{needs.discover_target_boards.outputs.targets}}
 
   Post_fail_to_Teams:
     needs: [verify_changes, build_samples, build_and_run_tests]

.github/workflows/on-pr.yml

@@ -60,14 +60,36 @@ jobs:
           path: |
             PR_number
 
+  discover_target_boards:
+    runs-on: ubuntu-latest
+    outputs:
+      targets: ${{ steps.platforms.outputs.targets }}
+    steps:
+      - name: install yq
+        run: pip install yq
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          path: sidewalk
+
+      - name: get_platforms
+        id: platforms
+        run: |
+          result=$(yq '.common.platform_allow' sidewalk/samples/sid_end_device/sample.yaml -c)
+          echo "targets=$result" >> $GITHUB_OUTPUT;
+          echo "$result"
+
   build_samples:
-    needs: Run_Configuration
+    needs: [Run_Configuration, discover_target_boards]
     uses: ./.github/workflows/samples_build.yml
     with:
       nrf_revision: ${{needs.Run_Configuration.outputs.NRF_revision}}
+      boards_to_test: ${{needs.discover_target_boards.outputs.targets}}
 
   build_and_run_tests:
-    needs: Run_Configuration
+    needs: [Run_Configuration, discover_target_boards]
     uses: ./.github/workflows/run_tests.yml
     with:
       nrf_revision: ${{needs.Run_Configuration.outputs.NRF_revision}}
+      boards_to_test: ${{needs.discover_target_boards.outputs.targets}}

.github/workflows/run_tests.yml

@@ -8,88 +8,10 @@ on:
         default: "main"
       boards_to_test:
         type: string
-        required: false
-        default: '["nrf52840dk/nrf52840", "nrf5340dk/nrf5340/cpuapp", "nrf54l15dk/nrf54l15/cpuapp"]'
+        required: true
+        default: '["nrf52840dk/nrf52840", "nrf5340dk/nrf5340/cpuapp", "nrf54l15dk/nrf54l15/cpuapp", "nrf54l15dk/nrf54l15/cpuapp/ns", "nrf54l15dk/nrf54l10/cpuapp"]'
 
 jobs:
-  build_manual_tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        subset: [1, 2, 3, 4, 5]
-    env:
-      MAX_SUBSETS: 5
-
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/nrfconnect/sdk-sidewalk:main
-      options: --cpus 2
-    defaults:
-      run:
-        shell: nrfutil toolchain-manager launch --install-dir /root/ncs bash -- {0}
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          path: sidewalk
-
-      - name: update NRF
-        run: |
-          cd sidewalk &&  git rev-parse HEAD > /workdir/sidewalk_hash && cd .. &&
-          west init -m https://github.com/nrfconnect/sdk-nrf --mr main . &&
-          cd nrf &&
-          git fetch origin ${{inputs.nrf_revision}}:PR-branch &&
-          git checkout PR-branch &&
-          cd .. &&
-          west config manifest.group-filter +sidewalk &&
-          west update -n -o=--depth=1 --path-cache /workdir/ &&
-          cd sidewalk && git checkout `cat /workdir/sidewalk_hash` && rm -rf /workdir/sidewalk_hash
-
-      - name: Install Python dependencies
-        run: |
-          python3 -m pip install -r nrf/scripts/requirements.txt -r zephyr/scripts/requirements.txt -r sidewalk/scripts/ci/requirements.txt
-
-      - name: Twister build manual tests
-        run: |
-          source zephyr/zephyr-env.sh
-          west twister --testsuite-root sidewalk/tests/manual --inline-logs --overflow-as-errors --enable-size-report --show-footprint --footprint-from-buildlog -vvv --build-only --subset ${{ matrix.subset }}/${{ env.MAX_SUBSETS }}
-
-      - name: Prepare artifacts for upload
-        run: |
-          mkdir -p subsets/${{ matrix.subset }}_of_${{ env.MAX_SUBSETS }}
-          cp twister-out/twister.json subsets/${{ matrix.subset }}_of_${{ env.MAX_SUBSETS }}
-
-      - name: Upload build artifacts
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: manual-tests-artifacts-${{ matrix.subset }}
-          path: |
-            subsets
-            twister-out/**/*sidewalk*/build.log
-            twister-out/**/*sidewalk*/zephyr/.config
-            twister-out/**/*sidewalk*/zephyr/runners.yaml
-            twister-out/**/*sidewalk*/zephyr/zephyr.elf
-            twister-out/**/*sidewalk*/zephyr/zephyr.hex
-            twister-out/**/*sidewalk*/zephyr/merged.hex
-            twister-out/**/*sidewalk*/zephyr/merged_domains.hex
-            twister-out/**/*sidewalk*/zephyr/*.dts
-            twister-out/**/*sidewalk*/zephyr/dfu_application.zip
-
-  merge_manual_tests:
-    needs: build_manual_tests
-    runs-on: ubuntu-latest
-    permissions: write-all
-    steps:
-      - name: Merge Artifacts
-        uses: actions/upload-artifact/merge@v4
-        with:
-          delete-merged: true
-          name: manual-tests-artifacts
-          pattern: manual-tests-artifacts-*
-
   build_and_run_x86_tests:
     runs-on: ubuntu-latest
     container:
@@ -125,7 +47,7 @@ jobs:
       - name: Twister build and run x86 tests
         run: |
           source zephyr/zephyr-env.sh
-          west twister --platform native_posix --platform unit_testing --coverage --coverage-basedir sidewalk --coverage-formats html,xml --enable-ubsan --enable-lsan --enable-asan --testsuite-root sidewalk --inline-logs --overflow-as-errors -vvv
+          west twister --test-config sidewalk/test_config.yaml --level="Github_tests" --testsuite-root sidewalk --platform native_sim --platform native_posix --platform unit_testing --coverage --coverage-basedir sidewalk --coverage-formats html,xml --enable-ubsan --enable-lsan --enable-asan --inline-logs --overflow-as-errors -vvv
 
       - name: Upload test results
         uses: actions/upload-artifact@v4

.github/workflows/run_tests_dut.yml

@@ -63,7 +63,7 @@ jobs:
       - name: Build DUT test artifacts
         run: |
           source zephyr/zephyr-env.sh
-          west twister --platform ${{inputs.target_board}} --testsuite-root sidewalk --filter runnable --shuffle-tests --shuffle-tests-seed 123 --inline-logs --overflow-as-errors -vvv --prep-artifacts-for-testing --package-artifacts PACKAGE_ARTIFACTS_${{ inputs.subset }}.tar.bz2 --subset ${{ inputs.subset }}/${{ inputs.max_subsets }}
+          west twister --test-config sidewalk/test_config.yaml --levels="Github_tests" --platform ${{inputs.target_board}} --testsuite-root sidewalk --filter runnable --shuffle-tests --shuffle-tests-seed 123 --inline-logs --overflow-as-errors -vvv --prep-artifacts-for-testing --package-artifacts PACKAGE_ARTIFACTS_${{ inputs.subset }}.tar.bz2 --subset ${{ inputs.subset }}/${{ inputs.max_subsets }}
 
       - name: Replace slash with underscore
         id: replace

.github/workflows/samples_build.yml

@@ -10,15 +10,21 @@ on:
         type: boolean
         required: false
         default: true
+      boards_to_test:
+        type: string
+        required: true
+        default: '["nrf52840dk/nrf52840", "nrf5340dk/nrf5340/cpuapp", "nrf54l15dk/nrf54l15/cpuapp", "nrf54l15dk/nrf54l15/cpuapp/ns", "nrf54l15dk/nrf54l10/cpuapp"]'
+
 
 jobs:
   build_samples:
     strategy:
       fail-fast: false
       matrix:
-        subset: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+        subset: [1, 2, 3, 4, 5]
+        board: ${{ fromJson(inputs.boards_to_test) }}
     env:
-      MAX_SUBSETS: 15
+      MAX_SUBSETS: 5
 
     runs-on: ubuntu-latest
     container:
@@ -51,21 +57,28 @@ jobs:
         run: |
           python3 -m pip install -r nrf/scripts/requirements.txt -r zephyr/scripts/requirements.txt -r sidewalk/scripts/ci/requirements.txt
 
+      - name: Replace slash with underscore
+        id: replace
+        run: |
+          TARGET_BOARD=${{ matrix.board  }}
+          TARGET_BOARD=${TARGET_BOARD//\//_}
+          echo "target_board=$TARGET_BOARD" >> $GITHUB_OUTPUT
+
       - name: Twister build samples
         run: |
           source zephyr/zephyr-env.sh
-          west twister --testsuite-root sidewalk/samples/ --inline-logs --overflow-as-errors --enable-size-report --show-footprint -vvv --build-only --subset ${{ matrix.subset }}/${{ env.MAX_SUBSETS }}
+          west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="samples" --platform ${{ matrix.board }} --inline-logs --overflow-as-errors --enable-size-report --show-footprint -vvv --build-only --subset ${{ matrix.subset }}/${{ env.MAX_SUBSETS }}
 
       - name: Prepare artifacts for upload
         run: |
-          mkdir -p subsets/${{ matrix.subset }}_of_${{ env.MAX_SUBSETS }}
-          cp twister-out/twister.json subsets/${{ matrix.subset }}_of_${{ env.MAX_SUBSETS }}
+          mkdir -p subsets/${{ matrix.subset }}_of_${{ env.MAX_SUBSETS }}_${{ steps.replace.outputs.target_board}}
+          cp twister-out/twister.json subsets/${{ matrix.subset }}_of_${{ env.MAX_SUBSETS }}_${{ steps.replace.outputs.target_board}}
 
       - name: Upload build metadata
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: sample-artifacts-meta-${{ matrix.subset }}
+          name: sample-artifacts-meta-${{ matrix.subset }}-${{ steps.replace.outputs.target_board}}
           path: |
             subsets
             twister-out/**/build.log
@@ -77,12 +90,14 @@ jobs:
         uses: actions/upload-artifact@v4
         if: ${{ inputs.save_images }}
         with:
-          name: sample-artifacts-build-${{ matrix.subset }}
+          name: sample-artifacts-build-${{ matrix.subset }}-${{ steps.replace.outputs.target_board}}
           path: |
+            subsets
             twister-out/**/zephyr.elf
             twister-out/**/zephyr.hex
             twister-out/**/merged.hex
             twister-out/**/merged_domains.hex
+            twister-out/**/dfu_multi_image.bin
             twister-out/**/dfu_application.zip
 
   merge_sample_artifacts:

samples/sid_end_device/sample.yaml

@@ -17,44 +17,44 @@ common:
     - nrf54l15dk/nrf54l15/cpuapp/ns
     - nrf54l15dk/nrf54l10/cpuapp
 tests:
-  sample.sidewalk.hello:
+  sidewalk.samples.sid_end_device.hello:
     extra_configs:
       - CONFIG_SID_END_DEVICE_PERSISTENT_LINK_MASK=y
       - CONFIG_SIDEWALK_FILE_TRANSFER=y
       - CONFIG_SIDEWALK_APPLICATION_NAME="hello"
     tags: Sidewalk hello
 
-  sample.sidewalk.hello.release:
+  sidewalk.samples.sid_end_device.hello.release:
     extra_args:
       FILE_SUFFIX=release
     extra_configs:
       - CONFIG_SID_END_DEVICE_PERSISTENT_LINK_MASK=y
       - CONFIG_SIDEWALK_APPLICATION_NAME="hello.release"
     tags: Sidewalk hello
 
-  sample.sidewalk.hello.ble_only:
+  sidewalk.samples.sid_end_device.hello.ble_only:
     extra_configs:
       - CONFIG_SIDEWALK_SUBGHZ_SUPPORT=n
       - CONFIG_SIDEWALK_APPLICATION_NAME="hello.ble_only"
     tags: Sidewalk hello BLE
 
-  sample.sidewalk.hello.ble_only.release:
+  sidewalk.samples.sid_end_device.hello.ble_only.release:
     extra_args:
       FILE_SUFFIX=release
     extra_configs:
       - CONFIG_SIDEWALK_SUBGHZ_SUPPORT=n
       - CONFIG_SIDEWALK_APPLICATION_NAME="hello.ble_only.release"
     tags: Sidewalk hello BLE
 
-  sample.sidewalk.demo:
+  sidewalk.samples.sid_end_device.demo:
     extra_args:
       OVERLAY_CONFIG="overlay-demo.conf"
     extra_configs:
       - CONFIG_SID_END_DEVICE_PERSISTENT_LINK_MASK=y
       - CONFIG_SIDEWALK_APPLICATION_NAME="demo"
     tags: Sidewalk demo
 
-  sample.sidewalk.demo.ble_only:
+  sidewalk.samples.sid_end_device.demo.ble_only:
     platform_allow:
       - thingy53/nrf5340/cpuapp
     extra_args:
@@ -66,22 +66,22 @@ tests:
       - thingy53/nrf5340/cpuapp
     tags: Sidewalk demo BLE
 
-  sample.sidewalk.dut:
+  sidewalk.samples.sid_end_device.dut:
     extra_args:
       OVERLAY_CONFIG="overlay-dut.conf"
     extra_configs:
       - CONFIG_SIDEWALK_APPLICATION_NAME="dut"
     tags: Sidewalk cli
 
-  sample.sidewalk.dut.ble_only:
+  sidewalk.samples.sid_end_device.dut.ble_only:
     extra_args:
       OVERLAY_CONFIG="overlay-dut.conf"
     extra_configs:
       - CONFIG_SIDEWALK_SUBGHZ_SUPPORT=n
       - CONFIG_SIDEWALK_APPLICATION_NAME="dut.ble_only"
     tags: Sidewalk cli BLE
 
-  sample.sidewalk.dut.no_secure:
+  sidewalk.samples.sid_end_device.dut.no_secure:
     extra_args:
       OVERLAY_CONFIG="overlay-dut.conf"
     extra_configs:

test_config.yaml

@@ -0,0 +1,36 @@
+platforms:
+  override_default_platforms: false
+  increased_platform_scope: false
+
+levels:
+  - name: samples
+    description: >
+      Prepare samples.
+    adds:
+      - sidewalk.samples.*
+      - sidewalk.test.acceptance.simple_bootloader
+
+  - name: unit
+    description: >
+      Check units in isolation.
+    adds:
+      - sidewalk.test.unit.*
+
+  - name: integration
+    description: >
+      Verify implementation of PAL on target
+    adds:
+      - sidewalk.test.integration.*
+
+  - name: Github_tests
+    description: >
+      set of tests run on GitHub
+    inherits:
+      - unit
+      - integration
+
+  - name: bugs
+    description: >
+      Test known issues and bug scenarios.
+    adds:
+      - sidewalk.test.acceptance.bugs.*

tests/functional/critical_region/testcase.yaml

@@ -1,5 +1,5 @@
 tests:
-  sidewalk.functional.critical_region:
+  sidewalk.test.integration.critical_region:
     sysbuild: true
     platform_allow:
       - nrf52840dk/nrf52840

tests/functional/crypto/testcase.yaml

@@ -1,5 +1,5 @@
 tests:
-  sidewalk.functional.crypto:
+  sidewalk.test.integration.crypto:
     sysbuild: true
     tags: Sidewalk
     platform_allow:

tests/functional/crypto_keys/testcase.yaml

@@ -1,5 +1,5 @@
 tests:
-  sidewalk.functional.crypto_keys:
+  sidewalk.test.integration.crypto_keys:
     sysbuild: true
     tags: Sidewalk
     platform_allow: