diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 222eeb2f56..f270351e8d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,12 +1,6 @@ ## CI parameters ```yaml -Github_actions: - #(branch, hash, pull/XXX/head) - NRF_revision: main - - # Do not change after creating PR - Create_NRF_PR: false Jenkins: test-sdk-sidewalk: master # To reconfigure functional tests: diff --git a/.github/workflows/nrf-upmerge-pr.yml b/.github/workflows/nrf-upmerge-pr.yml new file mode 100644 index 0000000000..9c1288f5da --- /dev/null +++ b/.github/workflows/nrf-upmerge-pr.yml @@ -0,0 +1,46 @@ +name: NRF upmerge + +on: + + workflow_dispatch: + +jobs: + change_nrf_revision: + name: Create PR with NRF upmerge + runs-on: ubuntu-latest + env: + CI_COMMIT_AUTHOR: Continuous Integration + steps: + - name: Checkout sidewalk + uses: actions/checkout@v3 + with: + path: sidewalk + ref: nrf_upmerge + + - name: Checkout nrf + uses: actions/checkout@v3 + with: + repository: nrfconnect/sdk-nrf + path: nrf + ref: main + fetch-depth: 0 + + - name: Change nrf revision + run: | + python3 -m pip install -r sidewalk/scripts/ci/requirements.txt + nrf_hash="$(git -C nrf rev-parse HEAD)" + python3 sidewalk/scripts/ci/replace_nrf_revision_in_west.py -r $nrf_hash sidewalk/west.yml + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + token: ${{secrets.NCS_GITHUB_TOKEN}} + path: sidewalk + commit-message: "NRF: upmerge\n\nupdate nrf revision in west.yml" + title: "NRF upmerge" + body: "PR created automatically by GitHub actions to update revision of NRF in west.yml" + delete-branch: true + branch: nrf_upmerge_PR + signoff: true + committer: Sidewalk Team + author: Sidewalk Team diff --git a/.github/workflows/on-commit.yml b/.github/workflows/on-commit.yml index 8f2ecb0082..66990405f6 100644 --- a/.github/workflows/on-commit.yml +++ b/.github/workflows/on-commit.yml @@ -33,37 +33,13 @@ jobs: with: reference_commit: "HEAD~${{needs.get_base_commit.outputs.commit_number}}" - discover_target_boards: - runs-on: ubuntu-24.04 - outputs: - targets: ${{ steps.platforms.outputs.targets }} - steps: - - name: install yq - run: pip install yq - - - name: Checkout - uses: actions/checkout@v4 - with: - path: sidewalk - - - name: get_platforms - id: platforms - run: | - result=$(yq '.common.platform_allow' sidewalk/samples/sid_end_device/sample.yaml -c) - echo "targets=$result" >> $GITHUB_OUTPUT; - echo "$result" - build_samples: - needs: [discover_target_boards] + name: Build samples uses: ./.github/workflows/samples_build.yml - with: - boards_to_test: ${{needs.discover_target_boards.outputs.targets}} build_and_run_tests: - needs: [discover_target_boards] + name: Build and run unit tests uses: ./.github/workflows/run_tests.yml - with: - boards_to_test: ${{needs.discover_target_boards.outputs.targets}} Post_fail_to_Teams: needs: [verify_changes, build_samples, build_and_run_tests] diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 14d39a3b40..af3951aac5 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -3,43 +3,6 @@ on: pull_request: jobs: - Run_Configuration: - runs-on: ubuntu-24.04 - outputs: - NRF_revision: ${{ steps.config.outputs.NRF_revision }} - steps: - - name: Read body of PR - env: - GH_TOKEN: ${{ github.token }} - run: | - gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /repos/${{github.repository}}/pulls/${{github.event.number}} --jq '.body' > pr_body.md - - - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - name: install yq - run: pip install yq - - - name: Get configuration - id: config - shell: bash {0} - run: | - csplit pr_body.md /\`\`\`yaml/1 - csplit xx01 /\`\`\`/ - mv xx00 configuration.yaml - cat configuration.yaml - revision=$(yq '.Github_actions.NRF_revision' configuration.yaml) - if [ $? != 0 ]; then - echo "Failed to read configuration use default main for NRF_revision" - echo "NRF_revision=main" >> $GITHUB_OUTPUT; - else - echo "NRF_revision=$revision" >> $GITHUB_OUTPUT; - fi; - - - name: print nrf revision - run: | - echo ${{ steps.config.outputs.NRF_revision }} verify_PR: uses: ./.github/workflows/validate_code_style.yml @@ -61,36 +24,10 @@ jobs: PR_number include-hidden-files: true - discover_target_boards: - runs-on: ubuntu-24.04 - outputs: - targets: ${{ steps.platforms.outputs.targets }} - steps: - - name: install yq - run: pip install yq - - - name: Checkout - uses: actions/checkout@v4 - with: - path: sidewalk - - - name: get_platforms - id: platforms - run: | - result=$(yq '.common.platform_allow' sidewalk/samples/sid_end_device/sample.yaml -c) - echo "targets=$result" >> $GITHUB_OUTPUT; - echo "$result" - build_samples: - needs: [Run_Configuration, discover_target_boards] + name: Build samples uses: ./.github/workflows/samples_build.yml - with: - nrf_revision: ${{needs.Run_Configuration.outputs.NRF_revision}} - boards_to_test: ${{needs.discover_target_boards.outputs.targets}} build_and_run_tests: - needs: [Run_Configuration, discover_target_boards] + name: Build and run unit tests uses: ./.github/workflows/run_tests.yml - with: - nrf_revision: ${{needs.Run_Configuration.outputs.NRF_revision}} - boards_to_test: ${{needs.discover_target_boards.outputs.targets}} diff --git a/.github/workflows/on-pr_nrf_manifest_update_PR.yml b/.github/workflows/on-pr_nrf_manifest_update_PR.yml deleted file mode 100644 index 43ea244a9a..0000000000 --- a/.github/workflows/on-pr_nrf_manifest_update_PR.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: handle manifest PR -on: - pull_request_target: - types: [opened, synchronize, closed] - branches: - - main - -jobs: - read_pr_configuration: - runs-on: ubuntu-24.04 - outputs: - Create_NRF_PR: ${{ steps.config.outputs.Create_NRF_PR }} - steps: - - name: Print Github context - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - run: echo "$GITHUB_CONTEXT" - - - name: Read body of PR - env: - GH_TOKEN: ${{ github.token }} - run: | - gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /repos/${{github.repository}}/pulls/${{github.event.number}} --jq '.body' > pr_body.md - - - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - name: install yq - run: pip install yq - - - name: Get configuration - id: config - shell: bash {0} - run: | - csplit pr_body.md /\`\`\`yaml/1 - csplit xx01 /\`\`\`/ - mv xx00 configuration.yaml - cat configuration.yaml - create_pr=$(yq '.Github_actions.Create_NRF_PR' configuration.yaml) - if [ $? != 0 ]; then - echo "Did not found Create_NRF_PR in configuration, default to false." - echo "Create_NRF_PR=false" >> $GITHUB_OUTPUT; - else - echo "Create_NRF_PR=$create_pr" >> $GITHUB_OUTPUT; - fi; - - create-manifest-pr: - runs-on: ubuntu-24.04 - needs: read_pr_configuration - if: ${{ fromJson(needs.read_pr_configuration.outputs.Create_NRF_PR)}} - steps: - - name: Create manifest PR - uses: nrfconnect/action-manifest-pr@main - with: - token: ${{ secrets.NCS_GITHUB_TOKEN }} diff --git a/.github/workflows/on_docker_change.yml b/.github/workflows/on_docker_change.yml index 17b743f4f2..de5c610a22 100644 --- a/.github/workflows/on_docker_change.yml +++ b/.github/workflows/on_docker_change.yml @@ -53,10 +53,7 @@ jobs: - name: Get revision of nrf to use id: revision run: | - REF_NAME="${{ github.ref_name }}" - if [ -z "$REF_NAME" ]; then - REF_NAME="main" # Set your default value here - fi + REF_NAME=$(yq '.manifest.projects | map(select(.name == "nrf"))[0].revision' west.yml) echo "REF_NAME=$REF_NAME" >> $GITHUB_OUTPUT echo "Ref name: $REF_NAME" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 9d17f796d2..f2a356a38d 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -2,17 +2,71 @@ name: Build Sidewalk samples on: workflow_call: inputs: - nrf_revision: - type: string + jobs_per_subset: + type: number required: false - default: "main" - boards_to_test: - type: string - required: true - default: '["nrf52840dk/nrf52840", "nrf5340dk/nrf5340/cpuapp", "nrf54l15dk/nrf54l15/cpuapp", "nrf54l15dk/nrf54l15/cpuapp/ns", "nrf54l15dk/nrf54l10/cpuapp"]' + default: 6 jobs: + discover_targets: + runs-on: self-hosted + container: + image: ghcr.io/nrfconnect/sdk-sidewalk:main + options: --cpus 2 --privileged + outputs: + total_platforms_json: ${{ steps.subsets.outputs.total_platforms_json }} + subset_matrix: ${{ steps.subsets.outputs.matrix }} + subset_config: ${{ steps.subsets.outputs.config }} + defaults: + run: + shell: nrfutil toolchain-manager launch --install-dir /root/ncs bash -- {0} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: sidewalk + + - name: update NRF + run: | + rm -rf .west; + west init -l sidewalk --mf west.yml && + west update -n -o=--depth=1 --path-cache /workdir/ + + - name: subsets configuration + id: subsets + run: | + rm -rf twister-out* max max_sorted configuration + west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="Github_tests" --exclude-platform native_sim --exclude-platform native_posix --exclude-platform unit_testing --dry-run + total_platforms=$(jq -r '.testsuites |map({platform: .platform}) | unique | .[].platform' twister-out/testplan.json) + total_platforms_json=$(printf '%s' "${total_platforms[@]}" | jq -R . | jq -s -c .) + echo "{}" > configuration + for platform in $total_platforms; do + west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="Github_tests" --dry-run --platform $platform + total_jobs=$(jq '.testsuites | length' twister-out/testplan.json) + subsets=$(( ( total_jobs + ${{inputs.jobs_per_subset}}) / ${{inputs.jobs_per_subset}})) + echo $subsets >> max + cat configuration | jq -c --arg platform_name $platform --arg subsets $subsets '. += {($platform_name): $subsets|tonumber }' > configuration_new + mv configuration_new configuration + done + sort max > max_sorted + max_subset=$(tail -n 1 max_sorted) + subset_matrix="[$(seq -s ', ' 1 $max_subset)]" + cat configuration + echo $subset_matrix + + echo "total_platforms_json=$total_platforms_json" >> $GITHUB_OUTPUT; + echo "matrix=$subset_matrix" >> $GITHUB_OUTPUT; + echo "config=`cat configuration`" >> $GITHUB_OUTPUT; + + - name: clean + if: always() + run: | + rm -rf twister* + build_and_run_x86_tests: + name: Run UT on x86 runs-on: ubuntu-24.04 container: image: ghcr.io/nrfconnect/sdk-sidewalk:main @@ -30,16 +84,9 @@ jobs: - name: update NRF run: | - cd sidewalk && git rev-parse HEAD > /workdir/sidewalk_hash && cd .. && - cp -r /workdir/nrf nrf; - west init -l nrf && - cd nrf && - git fetch origin ${{inputs.nrf_revision}}:PR-branch && - git checkout PR-branch && - cd .. && - west config manifest.group-filter +sidewalk && - west update -n -o=--depth=1 --path-cache /workdir/ && - cd sidewalk && git checkout `cat /workdir/sidewalk_hash` && rm -rf /workdir/sidewalk_hash + rm -rf .west; + west init -l sidewalk --mf west.yml && + west update -n -o=--depth=1 --path-cache /workdir/ - name: Install Python dependencies run: | @@ -84,18 +131,18 @@ jobs: build-and-test_on_hw: + needs: [discover_targets] strategy: fail-fast: false matrix: - subset: [1, 2] - board: ${{ fromJson(inputs.boards_to_test) }} + subset: ${{ fromJson(needs.discover_targets.outputs.subset_matrix) }} + target: ${{ fromJson(needs.discover_targets.outputs.total_platforms_json) }} uses: ./.github/workflows/run_tests_dut.yml with: - subset: ${{ matrix.subset }} - max_subsets: 2 run_tests: true - target_board: ${{ matrix.board }} - nrf_revision: ${{ inputs.nrf_revision }} + target: ${{matrix.target}} + subset: ${{matrix.subset}} + subset_config: '${{needs.discover_targets.outputs.subset_config}}' merge_tests_on_hw: needs: build-and-test_on_hw diff --git a/.github/workflows/run_tests_cli.yml b/.github/workflows/run_tests_cli.yml index bc3555ce0d..56a18f33b2 100644 --- a/.github/workflows/run_tests_cli.yml +++ b/.github/workflows/run_tests_cli.yml @@ -13,7 +13,7 @@ on: jobs: run_dut_ut: - name: "Run DUT tests (${{inputs.target_board}} - ${{ inputs.subset }})" + name: "Run DUT tests (${{inputs.target_board}})" runs-on: [self-hosted, linux, x64, "${{inputs.target_board}}"] container: image: ghcr.io/nrfconnect/sdk-sidewalk:main @@ -27,24 +27,25 @@ jobs: shell: nrfutil toolchain-manager launch --install-dir /root/ncs bash -- {0} steps: + - name: clean artifacts + if: always() + run: | + rm -rf pytest-sidewalk + rm -rf twister-out* + rm -rf sidewalk + rm -rf .west + - name: Checkout uses: actions/checkout@v4 with: + fetch-depth: 0 path: sidewalk - name: update NRF run: | - rm -rf .west nrf; - cd sidewalk && git rev-parse HEAD > /workdir/sidewalk_hash && cd .. - west init -m https://github.com/nrfconnect/sdk-nrf --mr main . && - cd nrf && - git fetch origin ${{inputs.nrf_revision}}:branch_to_run && - git checkout branch_to_run && - git rev-parse HEAD && - cd .. && - west config manifest.group-filter +sidewalk && - west update -n -o=--depth=1 --path-cache /workdir/ && - cd sidewalk && git checkout `cat /workdir/sidewalk_hash` && rm -rf /workdir/sidewalk_hash + rm -rf .west; + west init -l sidewalk --mf west.yml && + west update -n -o=--depth=1 --path-cache /workdir/ - name: Replace slash with underscore id: replace @@ -102,5 +103,7 @@ jobs: if: always() run: | rm -rf pytest-sidewalk - cd nrf; git checkout main; git branch -D branch_to_run; - rm -rf PACKAGE_ARTIFACTS_* twister-out + rm -rf twister-out* + rm -rf sidewalk_tmp + rm -rf sidewalk + rm -rf .west diff --git a/.github/workflows/run_tests_dut.yml b/.github/workflows/run_tests_dut.yml index e6445cfa1b..91c8c9d110 100644 --- a/.github/workflows/run_tests_dut.yml +++ b/.github/workflows/run_tests_dut.yml @@ -2,22 +2,15 @@ name: Build and run DUT tests on: workflow_call: inputs: - nrf_revision: + target: type: string - required: false - default: "main" - subset: - description: "" required: true - type: number - max_subsets: - description: "" - type: number - required: true - target_board: - description: "target board for tests" + subset_config: type: string required: true + subset: + type: number + required: true run_tests: description: "" required: false @@ -26,7 +19,8 @@ on: jobs: build_dut_tests: - name: "Build DUT tests (${{inputs.target_board}} - ${{ inputs.subset }})" + if: inputs.subset <= fromJson(inputs.subset_config)[inputs.target] + name: "Build ${{inputs.subset}}/${{fromJson(inputs.subset_config)[inputs.target]}} - ${{inputs.target}}" runs-on: ubuntu-24.04 container: image: ghcr.io/nrfconnect/sdk-sidewalk:main @@ -36,6 +30,14 @@ jobs: shell: nrfutil toolchain-manager launch --install-dir /root/ncs bash -- {0} steps: + - name: get_max_subset + id: max_subset + run: | + echo '${{inputs.subset_config}}' > configuration.json + max_subsets=$(jq --arg target_name "${{inputs.target}}" '.[($target_name)]' configuration.json) + echo "max_subset=$max_subsets" + echo "max_subset=$max_subsets" >> $GITHUB_OUTPUT + - name: Checkout uses: actions/checkout@v4 with: @@ -44,18 +46,9 @@ jobs: - name: update NRF run: | - rm -rf .west nrf; - cp -r /workdir/nrf nrf; - cd sidewalk && git rev-parse HEAD > /workdir/sidewalk_hash && cd .. && - west init -l nrf && - cd nrf && - git fetch origin ${{inputs.nrf_revision}}:branch_to_run && - git checkout branch_to_run && - git rev-parse HEAD && - cd .. && - west config manifest.group-filter +sidewalk && - west update -n -o=--depth=1 --path-cache /workdir/ && - cd sidewalk && git checkout `cat /workdir/sidewalk_hash` && rm -rf /workdir/sidewalk_hash + rm -rf .west; + west init -l sidewalk --mf west.yml && + west update -n -o=--depth=1 --path-cache /workdir/ - name: Install Python dependencies run: | @@ -64,7 +57,7 @@ jobs: - name: Replace slash with underscore id: replace run: | - TARGET_BOARD=${{ inputs.target_board }} + TARGET_BOARD=${{ inputs.target }} TARGET_BOARD=${TARGET_BOARD//\//_} echo "target_board=$TARGET_BOARD" >> $GITHUB_OUTPUT @@ -73,13 +66,13 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/ccache - key: ccache-tests_dut-${{ inputs.subset }}-${{steps.replace.outputs.target_board}}-${{ github.run_id }} - restore-keys: ccache-tests_dut-${{ inputs.subset }}-${{ steps.replace.outputs.target_board}}- + key: ccache-tests_dut-${{steps.replace.outputs.target_board}}-${{ github.run_id }} + restore-keys: ccache-tests_dut-${{ steps.replace.outputs.target_board}}- - name: Build DUT test artifacts run: | source zephyr/zephyr-env.sh - west twister --test-config sidewalk/test_config.yaml --level="Github_tests" --platform ${{inputs.target_board}} --testsuite-root sidewalk --shuffle-tests --shuffle-tests-seed 123 --inline-logs --overflow-as-errors -vvv --prep-artifacts-for-testing --package-artifacts PACKAGE_ARTIFACTS_${{ inputs.subset }}.tar.bz2 --subset ${{ inputs.subset }}/${{ inputs.max_subsets }} + west twister --test-config sidewalk/test_config.yaml --level="Github_tests" --platform ${{inputs.target}} --testsuite-root sidewalk --inline-logs --overflow-as-errors -vvv --prep-artifacts-for-testing --package-artifacts PACKAGE_ARTIFACTS_${{ inputs.subset }}.tar.bz2 --subset ${{ inputs.subset }}/${{steps.max_subset.outputs.max_subset}} - name: Print ccache stats run: | @@ -94,9 +87,9 @@ jobs: run_dut_ut: needs: [build_dut_tests] - if: ${{ fromJson(inputs.run_tests) }} - name: "Run DUT tests (${{inputs.target_board}} - ${{ inputs.subset }})" - runs-on: [self-hosted, linux, x64, "${{inputs.target_board}}"] + if: ${{ fromJson(inputs.run_tests) }} and ${{ inputs.subset <= fromJson(inputs.subset_config)[inputs.target]}} + name: "Run ${{inputs.subset}}/${{fromJson(inputs.subset_config)[inputs.target]}} - ${{inputs.target}}" + runs-on: [self-hosted, linux, x64, "${{inputs.target}}"] container: image: ghcr.io/nrfconnect/sdk-sidewalk:main options: --cpus 2 --privileged @@ -109,30 +102,29 @@ jobs: shell: nrfutil toolchain-manager launch --install-dir /root/ncs bash -- {0} steps: + - name: clean artifacts + if: always() + run: | + rm -rf PACKAGE_ARTIFACTS* twister-out* + rm -rf sidewalk + rm -rf .west + - name: Checkout uses: actions/checkout@v4 with: + fetch-depth: 0 path: sidewalk - name: update NRF run: | - rm -rf .west nrf; - cp -r /workdir/nrf nrf; - cd sidewalk && git rev-parse HEAD > /workdir/sidewalk_hash && cd .. && - west init -l nrf && - cd nrf && - git fetch origin ${{inputs.nrf_revision}}:branch_to_run && - git checkout branch_to_run && - git rev-parse HEAD && - cd .. && - west config manifest.group-filter +sidewalk && - west update -n -o=--depth=1 --path-cache /workdir/ && - cd sidewalk && git checkout `cat /workdir/sidewalk_hash` && rm -rf /workdir/sidewalk_hash + rm -rf .west; + west init -l sidewalk --mf west.yml && + west update -n -o=--depth=1 --path-cache /workdir/ - name: Replace slash with underscore id: replace run: | - TARGET_BOARD=${{ inputs.target_board }} + TARGET_BOARD=${{ inputs.target }} TARGET_BOARD=${TARGET_BOARD//\//_} echo "target_board=$TARGET_BOARD" >> $GITHUB_OUTPUT @@ -164,7 +156,7 @@ jobs: with: name: tests-dut_result_${{steps.replace.outputs.target_board}}_${{ inputs.subset }} path: | - twister-out/${{steps.replace.outputs.target_board}}_${{ inputs.subset }}_twister.xml + twister-out/${{steps.replace.outputs.target_board}}_twister.xml twister-out/**/handler.log twister-out/**/device.log twister-out/**/zephyr.hex @@ -174,5 +166,7 @@ jobs: - name: clean artifacts if: always() run: | - cd nrf; git checkout main; git branch -D branch_to_run; cd ..; - rm -rf PACKAGE_ARTIFACTS_* twister-out + rm -rf PACKAGE_ARTIFACTS* twister-out* + rm -rf sidewalk_tmp + rm -rf sidewalk + rm -rf .west diff --git a/.github/workflows/samples_build.yml b/.github/workflows/samples_build.yml index 62b793cca5..4c712e3efe 100644 --- a/.github/workflows/samples_build.yml +++ b/.github/workflows/samples_build.yml @@ -2,34 +2,26 @@ name: Build Sidewalk samples on: workflow_call: inputs: - nrf_revision: - type: string + jobs_per_subset: + type: number required: false - default: "main" + default: 4 save_images: type: boolean required: false default: true - boards_to_test: - type: string - required: false - default: '["nrf52840dk/nrf52840", "nrf5340dk/nrf5340/cpuapp", "nrf54l15dk/nrf54l15/cpuapp", "nrf54l15dk/nrf54l15/cpuapp/ns", "nrf54l15dk/nrf54l10/cpuapp"]' jobs: - build_samples: - strategy: - fail-fast: false - matrix: - subset: [1, 2, 3] - board: ${{ fromJson(inputs.boards_to_test) }} - env: - MAX_SUBSETS: 3 - - runs-on: ubuntu-24.04 + discover_targets: + runs-on: self-hosted container: image: ghcr.io/nrfconnect/sdk-sidewalk:main - options: --cpus 2 + options: --cpus 2 --privileged + outputs: + total_platforms_json: ${{ steps.subsets.outputs.total_platforms_json }} + subset_matrix: ${{ steps.subsets.outputs.matrix }} + subset_config: ${{ steps.subsets.outputs.config }} defaults: run: shell: nrfutil toolchain-manager launch --install-dir /root/ncs bash -- {0} @@ -43,83 +35,58 @@ jobs: - name: update NRF run: | - rm -rf .west nrf; - cp -r /workdir/nrf nrf; - cd sidewalk && git rev-parse HEAD > /workdir/sidewalk_hash && cd .. && - west init -l nrf && - cd nrf && - git fetch origin ${{inputs.nrf_revision}}:PR-branch && - git checkout PR-branch && - cd .. && - west config manifest.group-filter +sidewalk && - west update -n -o=--depth=1 --path-cache /workdir/ && - cd sidewalk && git checkout `cat /workdir/sidewalk_hash` && rm -rf /workdir/sidewalk_hash + rm -rf .west; + west init -l sidewalk --mf west.yml && + west update -n -o=--depth=1 --path-cache /workdir/ - - name: Install Python dependencies + - name: subsets configuration + id: subsets run: | - python3 -m pip install -r nrf/scripts/requirements.txt -r zephyr/scripts/requirements.txt -r sidewalk/scripts/ci/requirements.txt + rm -rf twister-out* max max_sorted configuration + west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="samples" --dry-run + total_platforms=$(jq -r '.testsuites |map({platform: .platform}) | unique | .[].platform' twister-out/testplan.json) + total_platforms_json=$(printf '%s' "${total_platforms[@]}" | jq -R . | jq -s -c .) + echo "{}" > configuration + for platform in $total_platforms; do + west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="samples" --dry-run --platform $platform + total_jobs=$(jq '.testsuites | length' twister-out/testplan.json) + subsets=$(( ( total_jobs + ${{inputs.jobs_per_subset}}) / ${{inputs.jobs_per_subset}})) + echo $subsets >> max + cat configuration | jq -c --arg platform_name $platform --arg subsets $subsets '. += {($platform_name): $subsets|tonumber }' > configuration_new + mv configuration_new configuration + done + sort max > max_sorted + max_subset=$(tail -n 1 max_sorted) + subset_matrix="[$(seq -s ', ' 1 $max_subset)]" + cat configuration + echo $subset_matrix - - name: Replace slash with underscore - id: replace - run: | - TARGET_BOARD=${{ matrix.board }} - TARGET_BOARD=${TARGET_BOARD//\//_} - echo "target_board=$TARGET_BOARD" >> $GITHUB_OUTPUT - - - name: Ccache - id: ccache - uses: actions/cache@v4 - with: - path: ~/.cache/ccache - key: ccache-samples-${{ matrix.subset }}-${{ steps.replace.outputs.target_board}}-${{ github.run_id }} - restore-keys: ccache-samples-${{ matrix.subset }}-${{ steps.replace.outputs.target_board}}- + echo "total_platforms_json=$total_platforms_json" >> $GITHUB_OUTPUT; + echo "matrix=$subset_matrix" >> $GITHUB_OUTPUT; + echo "config=`cat configuration`" >> $GITHUB_OUTPUT; - - name: Twister build samples - run: | - source zephyr/zephyr-env.sh - west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="samples" --platform ${{ matrix.board }} --inline-logs --overflow-as-errors --enable-size-report --show-footprint -vvv --build-only --subset ${{ matrix.subset }}/${{ env.MAX_SUBSETS }} - - - name: Print ccache stats - run: | - ccache -ps; - - - name: Prepare artifacts for upload + - name: clean + if: always() run: | - mkdir -p subsets/${{ matrix.subset }}_of_${{ env.MAX_SUBSETS }}_${{ steps.replace.outputs.target_board}} - cp twister-out/twister.json subsets/${{ matrix.subset }}_of_${{ env.MAX_SUBSETS }}_${{ steps.replace.outputs.target_board}} + rm -rf twister* sidewalk* max max_sorted configuration - - name: Upload build metadata - uses: actions/upload-artifact@v4 - if: always() - with: - name: sample-artifacts-meta-${{ matrix.subset }}-${{ steps.replace.outputs.target_board}} - path: | - subsets - twister-out/**/build.log - twister-out/**/.config - twister-out/**/runners.yaml - twister-out/**/*.dts - include-hidden-files: true + build_samples: + needs: [discover_targets] + strategy: + fail-fast: false + matrix: + subset: ${{ fromJson(needs.discover_targets.outputs.subset_matrix) }} + target: ${{ fromJson(needs.discover_targets.outputs.total_platforms_json) }} - - name: Upload build artifacts - uses: actions/upload-artifact@v4 - if: ${{ inputs.save_images }} - with: - name: sample-artifacts-build-${{ matrix.subset }}-${{ steps.replace.outputs.target_board}} - path: | - subsets - twister-out/**/zephyr.elf - twister-out/**/zephyr.hex - twister-out/**/merged.hex - twister-out/**/merged_domains.hex - twister-out/**/dfu_multi_image.bin - twister-out/**/dfu_application.zip - include-hidden-files: true + uses: ./.github/workflows/samples_build_target.yml + with: + target: ${{matrix.target}} + subset: ${{matrix.subset}} + subset_config: '${{needs.discover_targets.outputs.subset_config}}' merge_sample_artifacts: needs: build_samples runs-on: ubuntu-24.04 - if: always() permissions: write-all steps: - name: Merge Artifacts diff --git a/.github/workflows/samples_build_target.yml b/.github/workflows/samples_build_target.yml new file mode 100644 index 0000000000..a99432e993 --- /dev/null +++ b/.github/workflows/samples_build_target.yml @@ -0,0 +1,110 @@ +name: Build Sidewalk samples +on: + workflow_call: + inputs: + save_images: + type: boolean + required: false + default: true + target: + type: string + required: true + subset_config: + type: string + required: true + subset: + type: number + required: true + +jobs: + build_samples: + name: "Build ${{inputs.subset}}/${{fromJson(inputs.subset_config)[inputs.target]}} - ${{inputs.target}}" + if: inputs.subset <= fromJson(inputs.subset_config)[inputs.target] + runs-on: ubuntu-24.04 + container: + image: ghcr.io/nrfconnect/sdk-sidewalk:main + options: --cpus 2 + defaults: + run: + shell: nrfutil toolchain-manager launch --install-dir /root/ncs bash -- {0} + + steps: + - name: get_max_subset + id: max_subset + run: | + echo '${{inputs.subset_config}}' > configuration.json + max_subsets=$(jq --arg target_name "${{inputs.target}}" '.[($target_name)]' configuration.json) + echo "max_subset=$max_subsets" + echo "max_subset=$max_subsets" >> $GITHUB_OUTPUT + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: sidewalk + + - name: update NRF + run: | + rm -rf .west; + west init -l sidewalk --mf west.yml && + west update -n -o=--depth=1 --path-cache /workdir/ + + - name: Install Python dependencies + run: | + python3 -m pip install -r nrf/scripts/requirements.txt -r zephyr/scripts/requirements.txt -r sidewalk/scripts/ci/requirements.txt + + - name: Replace slash with underscore + id: replace + run: | + TARGET_BOARD=${{ inputs.target}} + TARGET_BOARD=${TARGET_BOARD//\//_} + echo "target_board=$TARGET_BOARD" >> $GITHUB_OUTPUT + + - name: Ccache + id: ccache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-samples-${{ steps.replace.outputs.target_board}}-${{ github.run_id }} + restore-keys: ccache-samples-${{ steps.replace.outputs.target_board}}- + + - name: Twister build samples + run: | + source zephyr/zephyr-env.sh + west twister --test-config sidewalk/test_config.yaml --testsuite-root sidewalk --level="samples" --platform ${{ inputs.target}} --inline-logs --overflow-as-errors --enable-size-report --show-footprint -vvv --build-only --subset ${{inputs.subset}}/${{steps.max_subset.outputs.max_subset}} + + - name: Print ccache stats + run: | + ccache -ps; + + - name: Prepare artifacts for upload + run: | + mkdir -p subsets/${{inputs.subset}}_of_${{ steps.max_subset.outputs.max_subset }}_${{ steps.replace.outputs.target_board}} + cp twister-out/twister.json subsets/${{inputs.subset}}_of_${{ steps.max_subset.outputs.max_subset }}_${{ steps.replace.outputs.target_board}} + + - name: Upload build metadata + uses: actions/upload-artifact@v4 + with: + name: sample-artifacts-meta-${{ steps.replace.outputs.target_board}}-${{inputs.subset}} + path: | + subsets + twister-out/**/build.log + twister-out/**/.config + twister-out/**/runners.yaml + twister-out/**/*.dts + include-hidden-files: true + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + if: ${{ inputs.save_images }} + with: + name: sample-artifacts-build-${{ steps.replace.outputs.target_board}}-${{inputs.subset}} + path: | + subsets + twister-out/**/zephyr.elf + twister-out/**/zephyr.hex + twister-out/**/merged.hex + twister-out/**/merged_domains.hex + twister-out/**/dfu_multi_image.bin + twister-out/**/dfu_application.zip + include-hidden-files: true diff --git a/.github/workflows/validate_code_style.yml b/.github/workflows/validate_code_style.yml index 16e732a025..b0ab923dfb 100644 --- a/.github/workflows/validate_code_style.yml +++ b/.github/workflows/validate_code_style.yml @@ -60,11 +60,9 @@ jobs: - name: update NRF run: | - cd sidewalk && git rev-parse HEAD > /workdir/sidewalk_hash; cd .. - west init -m https://github.com/nrfconnect/sdk-nrf --mr main . - west config manifest.group-filter +sidewalk + rm -rf .west; + west init -l sidewalk --mf west.yml && west update -n -o=--depth=1 --path-cache /workdir/ - cd sidewalk; git checkout `cat /workdir/sidewalk_hash`; rm -rf /workdir/sidewalk_hash - name: Run Compliance Tests run: | diff --git a/doc/includes/include_building_and_running.txt b/doc/includes/include_building_and_running.txt index ea574a5098..8197eeb7d0 100644 --- a/doc/includes/include_building_and_running.txt +++ b/doc/includes/include_building_and_running.txt @@ -3,7 +3,7 @@ This sample can be found under :file:`samples/sid_end_device`. .. note:: Before you flash your Sidewalk sample, make sure you have completed the following: - * You downloaded the Sidewalk repository and updated west according to the :ref:`dk_building_sample_app` section. + * You have set up the Sidewalk repository using the standalone west.yml configuration as described in the :ref:`setting_up_sdk` section. * You provisioned your device during the :ref:`setting_up_sidewalk_prototype`. This step needs to be completed only once. diff --git a/doc/index.rst b/doc/index.rst index e8310ed981..8dfa11f8a1 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -7,8 +7,8 @@ Be aware, that by downloading and including the Sidewalk repository, you accept the above-mentioned license. - Due to these restrictions, Sidewalk cannot be automatically fetched through the nRF Connect SDK and will require performing a dedicated setup. - All information regarding completing it is included in this guide, so ensure you follow it in order. + Due to these restrictions, Sidewalk is now managed through its own west.yml configuration, separate from the nRF Connect SDK manifest. + All information regarding completing the setup is included in this guide, so ensure you follow it in order. Welcome to the nRF Connect SDK - Amazon Sidewalk ################################################ diff --git a/doc/releases_and_migration.rst b/doc/releases_and_migration.rst index 7172eeb0f8..2fdd8a4227 100644 --- a/doc/releases_and_migration.rst +++ b/doc/releases_and_migration.rst @@ -25,3 +25,4 @@ See information about the latest release notes and the available migration guide releases_and_migration/migration_guide_v280.rst releases_and_migration/migration_guide_v260.rst + releases_and_migration/migration_guide_addon_v010.rst diff --git a/doc/releases_and_migration/migration_guide_addon_v010.rst b/doc/releases_and_migration/migration_guide_addon_v010.rst new file mode 100644 index 0000000000..2140dacd50 --- /dev/null +++ b/doc/releases_and_migration/migration_guide_addon_v010.rst @@ -0,0 +1,157 @@ +.. _migration_guide_addon_v010: + +Migration Guide for moving to Sidewalk Add-on +############################################ + +This guide explains how to migrate your Sidewalk development environment from using the NCS manifest to the new standalone west.yml setup. + +Overview +======== + +The Sidewalk repository has been moved out of the NCS manifest into its own west.yml configuration. This change provides more flexibility and better control over the Sidewalk development environment. The Sidewalk release cycle will be detached from NCS, allowing for faster releases of new libraries on stable versions of NCS. The Sidewalk repository is now visible in the NCS add-on index repository, which enables an alternative setup method using the VSCode NRF Connect add-on. + +Prerequisites +============ + +Before starting the migration, ensure you have: + +* Git installed on your system +* Python 3.8 or later installed +* The west tool installed +* nrfutil installed + +Migration Steps +============= + +1. Initialize Sidewalk Repository +------------------------------- + +There are three ways to initialize the Sidewalk repository: + +.. tabs:: + + .. tab:: Option 1: Direct Initialization (Recommended) + + .. code-block:: console + + # Initialize west with the remote manifest + west init -m https://github.com/nrfconnect/sdk-sidewalk + + .. tab:: Option 2: Manual Clone and Initialize + + .. code-block:: console + + # Clone the Sidewalk repository into sidewalk directory + git clone https://github.com/nordicsemiconductor/sidewalk.git sidewalk + # Initialize west with local manifest + west init -l sidewalk + + .. tab:: Option 3: Clean Existing Configuration + + If you already have an NCS setup with Sidewalk, you need to remove the existing west configuration first: + + .. code-block:: console + + # Remove the .west directory to clean the west configuration + rm -rf .west + # Checkout and pull main branch in Sidewalk repository + cd sidewalk + git checkout main + git pull origin main + cd .. + # Initialize west with local manifest + west init -l sidewalk + +2. Update Repositories +--------------------- + + .. code-block:: console + + # Update all repositories + west update + +3. Update Toolchain and Install Dependencies +------------------------------------------ + + .. code-block:: console + + # Get the toolchain hash and download it + nrf/scripts/toolchain.py + nrfutil toolchain install --ncs-version $(nrf/scripts/toolchain.py --ncs-version) + # Install Python dependencies for nRF and Zephyr + pip install -r nrf/scripts/requirements.txt + pip install -r zephyr/scripts/requirements.txt + # Install Sidewalk Python requirements + pip install -r requirements.txt + +Verification +=========== + +To verify your migration was successful: + +1. Check that all repositories are properly cloned: + + .. code-block:: console + + west list + + Expected output should show all required repositories with their paths and revisions. For example: + + .. code-block:: console + + name path revision + ---------------------- ---------------------- ---------------------- + manifest sidewalk HEAD + ... + + Invalid output would be: + * Empty list + * Missing repositories + * Error messages + * "Not found" status for any repository + +2. Verify the build process: + + .. code-block:: console + + west build -b nrf54l15dk/nrf54l15/cpuapp samples/sid_end_device + + This will build the Sidewalk end device sample application. If the build succeeds, your migration was successful. + +Common Issues and Solutions +========================= + +1. Repository Conflicts +---------------------- + + If you encounter repository conflicts during the migration: + + .. code-block:: console + + # Remove the .west directory + rm -rf .west + # Reinitialize west + west init -l sidewalk + # Update repositories + west update + +2. Build Errors +-------------- + + If you encounter build errors: + + * Ensure all dependencies are installed + * Check that the NCS environment is properly sourced + * Verify that you're using the correct board target + * Try building the Zephyr hello world sample to verify your basic build environment: + + .. code-block:: console + + west build -b nrf54l15dk/nrf54l15/cpuapp zephyr/samples/basic/hello_world + +Additional Resources +================== + +* :ref:`setting_up_environment` +* :ref:`samples_list` +* :ref:`compatibility_matrix` diff --git a/doc/setting_up_sidewalk_environment/setting_up_sdk.rst b/doc/setting_up_sidewalk_environment/setting_up_sdk.rst index 4d44af6bc7..c3f23e79eb 100644 --- a/doc/setting_up_sidewalk_environment/setting_up_sdk.rst +++ b/doc/setting_up_sidewalk_environment/setting_up_sdk.rst @@ -12,51 +12,53 @@ Once confirmed, follow the `Installing the nRF Connect SDK`_ instructions. .. _dk_building_sample_app: -Downloading the Sidewalk repository +Setting up the Sidewalk Repository *********************************** -Once you have installed the nRF Connect SDK, download the Sidewalk application: +The Sidewalk repository is now managed through its own west.yml configuration. To set up the Sidewalk development environment: +There are two ways to initialize the Sidewalk repository: -1. Open a terminal window in the ncs folder. - Your directory structure should look as follows: +.. tabs:: - .. code-block:: console - - . - |___ .west - |___ bootloader - |___ modules - |___ nrf - |___ nrfxlib - |___ zephyr - |___ ... + .. tab:: Option 1: Direct Initialization (Recommended) -#. Enable the Sidewalk group filter for west. + .. code-block:: console - .. code-block:: console + # Initialize west with the remote manifest + west init -m https://github.com/nrfconnect/sdk-sidewalk - $ west config manifest.group-filter "+sidewalk" + .. tab:: Option 2: Manual Clone and Initialize - Check for Sidewalk presence in west: + .. code-block:: console - .. code-block:: console + # Clone the Sidewalk repository into sidewalk directory + git clone https://github.com/nordicsemiconductor/sidewalk.git sidewalk + # Initialize west with local manifest + west init -l sidewalk - $ west list sidewalk - sidewalk sidewalk https://github.com/nrfconnect/sdk-sidewalk +.. note:: + If you are migrating from an existing NCS setup with Sidewalk, please refer to the :ref:`migration_guide_addon_v010` for detailed migration steps. -#. Update all repositories: +4. Update all repositories: .. code-block:: console - $ west update + west update Depending on your connection, the update might take some time. -#. Install Python requirements for Sidewalk. +5. Install the toolchain and update Python packages: .. code-block:: console - $ pip install -r sidewalk/requirements.txt + # Get the toolchain hash and download it + nrf/scripts/toolchain.py + nrfutil toolchain install --ncs-version $(nrf/scripts/toolchain.py --ncs-version) + # Install Python dependencies for nRF and Zephyr + pip install -r nrf/scripts/requirements.txt + pip install -r zephyr/scripts/requirements.txt + # Install Sidewalk Python requirements + pip install -r requirements.txt Extracting nRF Command Line Tools ********************************* diff --git a/scripts/ci/requirements.txt b/scripts/ci/requirements.txt index 203d13822a..cb652a9461 100644 --- a/scripts/ci/requirements.txt +++ b/scripts/ci/requirements.txt @@ -3,3 +3,4 @@ autopep8>=2.0.1 ruamel.yaml>=0.17.21 clang-format yamllint>=1.32.0 +yq>=3.4.3 diff --git a/west.yml b/west.yml new file mode 100644 index 0000000000..fc769cea4f --- /dev/null +++ b/west.yml @@ -0,0 +1,20 @@ +# +# Copyright (c) 2025 Nordic Semiconductor ASA +# +# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause +# + +manifest: + remotes: + - name: ncs + url-base: https://github.com/nrfconnect + + projects: + - name: nrf + remote: ncs + repo-path: sdk-nrf + revision: 10dd3ddbf2560aba2da99a2dc2536b177a1498d7 + import: true + + self: + path: sidewalk