nightly.yml

name: nightly

# This workflow picks up the latest changes of toolbox binaries
# and runs end-to-end tests to detect regressions. It is triggered by:
#   1. A schedule set for every midnight
#   2. Manual triggering of the workflow
#   3. Changes to the workflow file

on:
  workflow_dispatch:
  schedule:
    - cron: '0 2 * * *'
  pull_request:
    paths:
      - '.github/workflows/nightly.yml'
      - '.github/matrix_includes.json'
      - 'debian/**'
      - 'test/e2e/**'
      - 'test/vcpkg-configuration.json'
      - '!**/*.md'
env:
  retention-days: 7

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  matrix_prep:
    runs-on: ubuntu-latest
    if: github.repository == 'Open-CMSIS-Pack/cmsis-toolbox'
    outputs:
      build: ${{ steps.build-matrix.outputs.matrix }}
      test: ${{ steps.test-matrix.outputs.matrix }}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
        with:
          egress-policy: audit

      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

      - uses: JoshuaTheMiller/conditional-build-matrix@81b51eb8d89e07b86404934b5fecde1cea1163a5 # v2.0.1
        id: build-matrix
        with:
          inputFile: '.github/matrix_includes.json'
          filter: '[?runOnBranch==`always`]'

      - uses: JoshuaTheMiller/conditional-build-matrix@81b51eb8d89e07b86404934b5fecde1cea1163a5 # v2.0.1
        id: test-matrix
        with:
          inputFile: '.github/matrix_includes.json'
          filter: '[?runTests==`true`]'

  create-toolbox:
    name: Create toolbox [${{ matrix.target }}-${{ matrix.arch }}]
    needs: [ matrix_prep ]
    runs-on: ${{ matrix.runs_on }}
    continue-on-error: true
    strategy:
      matrix: ${{ fromJson(needs.matrix_prep.outputs.build) }}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
        with:
          egress-policy: audit

      - name: Checkout cmsis-toolbox repository
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          fetch-depth: 0

      - name: Authenticate with GitHub CLI
        shell: bash
        run: |
          echo ${{ secrets.GITHUB_TOKEN }} | gh auth login --with-token

      - name: Get latest nightly Run ID
        id: get_run_id
        shell: bash
        run: |
          RUN_ID=$(gh run list --limit 1 --workflow nightly --repo Open-CMSIS-Pack/devtools --json databaseId --jq '.[0].databaseId')
          echo "NIGHTLY_RUN_ID=$RUN_ID" >> $GITHUB_OUTPUT

      # Clean Go mod cache
      - name: Clean Go mod cache
        shell: bash
        run: |
          for dir in cbuild cpackget cbuild2cmake cbridge vidx2pidx; do
            rm -rf "${{ github.workspace }}/.gocache/$dir"
            mkdir -p "${{ github.workspace }}/.gocache/$dir"
          done

      # Download and build cbuild executable
      - name: Checkout cbuild repository
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          repository: Open-CMSIS-PACK/cbuild
          path: cbuild
          fetch-depth: 0

      - name: Install Go
        uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
        with:
          go-version-file: cbuild/go.mod
          cache: false

      - name: Build cbuild executable
        working-directory: cbuild
        shell: bash
        run: |
          export ABS_CBUILD_GOMODCACHE=$(realpath "${{ github.workspace }}/.gocache/cbuild")
          GOOS=${{ matrix.target }} \
          GOARCH=${{ matrix.arch }} \
          GOMODCACHE=$ABS_CBUILD_GOMODCACHE \
          go build -ldflags "-X main.version=$(git describe --tags)" \
            -o build/cbuild${{ matrix.binary_extension }} ./cmd/cbuild

      # Download and build cpackget executable
      - name: Checkout cpackget repository
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          repository: Open-CMSIS-PACK/cpackget
          path: cpackget
          fetch-depth: 0

      - name: Install Go
        uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
        with:
          go-version-file: cpackget/go.mod
          cache: false

      - name: Build cpackget executable
        working-directory: cpackget
        shell: bash
        run: |
          export ABS_CPACKGET_GOMODCACHE=$(realpath "${{ github.workspace }}/.gocache/cpackget")
          GOMODCACHE=${{ github.workspace }}/.gocache/cpackget \
          GOOS=${{ matrix.target }} \
          GOARCH=${{ matrix.arch }} \
          GOMODCACHE=$ABS_CPACKGET_GOMODCACHE \
          go build -ldflags "-X main.version=$(git describe --tags)" \
            -o build/cpackget${{ matrix.binary_extension }} ./cmd

      # Download and build cbuild2cmake executable
      - name: Checkout cbuild2cmake repository
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          repository: Open-CMSIS-PACK/cbuild2cmake
          path: cbuild2cmake
          fetch-depth: 0

      - name: Install Go
        uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
        with:
          go-version-file: cbuild2cmake/go.mod
          cache: false

      - name: Build cbuild2cmake executable
        working-directory: cbuild2cmake
        shell: bash
        run: |
          export ABS_CBUILD2CMAKE_GOMODCACHE=$(realpath "${{ github.workspace }}/.gocache/cbuild2cmake")
          GOOS=${{ matrix.target }} \
          GOARCH=${{ matrix.arch }} \
          GOMODCACHE=$ABS_CBUILD2CMAKE_GOMODCACHE \
          go build -ldflags "-X main.version=$(git describe --tags)" \
            -o build/cbuild2cmake${{ matrix.binary_extension }} ./cmd/cbuild2cmake

      # Download and build vidx2pidx executable
      - name: Checkout vidx2pidx repository
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          repository: Open-CMSIS-PACK/vidx2pidx
          path: vidx2pidx
          fetch-depth: 0

      - name: Install Go
        uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
        with:
          go-version-file: vidx2pidx/go.mod
          cache: false

      - name: Build vidx2pidx executable
        working-directory: vidx2pidx
        shell: bash
        run: |
          export ABS_VIDX2PIDX_GOMODCACHE=$(realpath "${{ github.workspace }}/.gocache/vidx2pidx")
          GOOS=${{ matrix.target }} \
          GOARCH=${{ matrix.arch }} \
          GOMODCACHE=$ABS_VIDX2PIDX_GOMODCACHE \
          go build -ldflags "-X main.version=$(git describe --tags)" \
            -o build/vidx2pidx${{ matrix.binary_extension }} ./cmd

      # Download and build cbridge executable
      - name: Checkout cbridge repository
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          repository: Open-CMSIS-PACK/generator-bridge
          path: cbridge
          fetch-depth: 0

      - name: Install Go
        uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
        with:
          go-version-file: cbridge/go.mod
          cache-dependency-path: cbridge/go.sum

      - name: Build cbridge executable
        working-directory: cbridge
        shell: bash
        run: |
          export ABS_CBRIDGE_GOMODCACHE=$(realpath "${{ github.workspace }}/.gocache/cbridge")
          GOOS=${{ matrix.target }} \
          GOARCH=${{ matrix.arch }} \
          GOMODCACHE=$ABS_CBRIDGE_GOMODCACHE \
          go build -ldflags "-X main.version=$(git describe --tags)" \
            -o build/cbridge${{ matrix.binary_extension }} ./cmd

      # Download projmgr and cbuildgen from nightly
      - name: Download Open-CMSIS-Pack/devtools nightly artifacts
        shell: bash
        run : |
          gh run download -D projmgr-${{ matrix.target }}-${{ matrix.arch }} ${{ steps.get_run_id.outputs.NIGHTLY_RUN_ID }} -n projmgr-${{ matrix.target }}-${{ matrix.arch }} -R Open-CMSIS-Pack/devtools
          gh run download -D cbuildgen-${{ matrix.target }}-${{ matrix.arch }} ${{ steps.get_run_id.outputs.NIGHTLY_RUN_ID }} -n cbuildgen-${{ matrix.target }}-${{ matrix.arch }} -R Open-CMSIS-Pack/devtools
          gh run download -D packchk-${{ matrix.target }}-${{ matrix.arch }} ${{ steps.get_run_id.outputs.NIGHTLY_RUN_ID }} -n packchk-${{ matrix.target }}-${{ matrix.arch }} -R Open-CMSIS-Pack/devtools
          gh run download -D svdconv-${{ matrix.target }}-${{ matrix.arch }} ${{ steps.get_run_id.outputs.NIGHTLY_RUN_ID }} -n svdconv-${{ matrix.target }}-${{ matrix.arch }} -R Open-CMSIS-Pack/devtools

      - name: Unzip files
        shell: bash
        run: |
          mkdir -p packchk
          mkdir -p svdconv
          packchk_dir="packchk-${{ matrix.target }}-${{ matrix.arch }}"
          svdconv_dir="svdconv-${{ matrix.target }}-${{ matrix.arch }}"
          if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "macOS" ]]; then
            tar -xvf ${packchk_dir}/packchk-*-${{ matrix.target }}-${{ matrix.arch }}.tbz2 -C packchk
            tar -xvf ${svdconv_dir}/svdconv-*-${{ matrix.target }}-${{ matrix.arch }}.tbz2 -C svdconv
          elif [[ "$RUNNER_OS" == "Windows" ]]; then
            unzip ${packchk_dir}/packchk-\*-${{ matrix.target }}-${{ matrix.arch }}.zip -d packchk
            unzip ${svdconv_dir}/svdconv-\*-${{ matrix.target }}-${{ matrix.arch }}.zip -d svdconv
          fi

      - name: Checkout Open-CMSIS-PACK/devtools repo
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          repository: Open-CMSIS-PACK/devtools
          path: devtools

      - name: Copy Files
        shell: bash
        run: |
          mkdir -p ./cmsis-toolbox/bin
          mkdir -p ./cmsis-toolbox/etc
          mkdir -p ./cmsis-toolbox/doc
          cp ./projmgr-${{ matrix.target }}-${{ matrix.arch }}/csolution${{ matrix.binary_extension }}      ./cmsis-toolbox/bin/csolution${{ matrix.binary_extension }}
          cp ./cbuildgen-${{ matrix.target }}-${{ matrix.arch }}/cbuildgen${{ matrix.binary_extension }}*   ./cmsis-toolbox/bin/cbuildgen${{ matrix.binary_extension }}
          cp ./packchk/packchk${{ matrix.binary_extension }}                                                ./cmsis-toolbox/bin/packchk${{ matrix.binary_extension }}
          cp ./svdconv/svdconv${{ matrix.binary_extension }}                                                ./cmsis-toolbox/bin/svdconv${{ matrix.binary_extension }}
          cp ./cbuild/build/cbuild${{ matrix.binary_extension }}                                            ./cmsis-toolbox/bin/cbuild${{ matrix.binary_extension }}
          cp ./cpackget/build/cpackget${{ matrix.binary_extension }}                                        ./cmsis-toolbox/bin/cpackget${{ matrix.binary_extension }}
          cp ./cbuild2cmake/build/cbuild2cmake${{ matrix.binary_extension }}                                ./cmsis-toolbox/bin/cbuild2cmake${{ matrix.binary_extension }}
          cp ./cbridge/build/cbridge${{ matrix.binary_extension }}                                          ./cmsis-toolbox/bin/cbridge${{ matrix.binary_extension }}
          cp ./cbridge/scripts/MCUXpresso_Config_Tools/${{ matrix.target }}-${{ matrix.arch }}/launch-MCUXpressoConfigTools*     ./cmsis-toolbox/bin/
          cp ./cbridge/scripts/Infineon_Device_Configurator/${{ matrix.target }}-${{ matrix.arch }}/launch-Infineon_Dev_Config*  ./cmsis-toolbox/bin/
          cp ./vidx2pidx/build/vidx2pidx${{ matrix.binary_extension }}                                      ./cmsis-toolbox/bin/vidx2pidx${{ matrix.binary_extension }}
          cp -r devtools/tools/projmgr/templates/*                ./cmsis-toolbox/etc
          cp -r devtools/tools/projmgr/schemas/*                  ./cmsis-toolbox/etc
          cp -r devtools/tools/buildmgr/cbuildgen/scripts/*       ./cmsis-toolbox/etc
          cp -r devtools/tools/buildmgr/cbuildgen/config/*        ./cmsis-toolbox/etc
          curl https://raw.githubusercontent.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/main/schema/PACK.xsd --output ./cmsis-toolbox/etc/PACK.xsd
          curl https://raw.githubusercontent.com/Open-CMSIS-Pack/Open-CMSIS-Pack-Spec/main/schema/PackIndex.xsd --output ./cmsis-toolbox/etc/PackIndex.xsd
          curl https://raw.githubusercontent.com/Open-CMSIS-Pack/debug-adapter-registry/refs/heads/main/registry/debug-adapters.yml --output ./cmsis-toolbox/etc/debug-adapters.yml
          curl https://raw.githubusercontent.com/Open-CMSIS-Pack/debug-adapter-registry/refs/heads/main/schemas/debug-adapters.schema.json --output ./cmsis-toolbox/etc/debug-adapters.schema.json
          cp ./docs/LICENSE.txt ./cmsis-toolbox
          cp ./docs/index.html ./cmsis-toolbox/doc

      - name: Update toolchain config files
        if: ${{ matrix.target == 'windows' }}
        shell: bash
        run: |
            sed -i '/set(EXT/c\  set(EXT .exe)' ./cmsis-toolbox/etc/AC6.6.16.2.cmake
            sed -i '/set(EXT/c\  set(EXT .exe)' ./cmsis-toolbox/etc/GCC.10.3.1.cmake
            sed -i '/set(EXT/c\  set(EXT .exe)' ./cmsis-toolbox/etc/IAR.9.32.1.cmake
            sed -i '/set(EXT/c\  set(EXT .exe)' ./cmsis-toolbox/etc/CLANG.17.0.1.cmake
            sed -i '/set(EXT/c\  set(EXT .exe)' ./cmsis-toolbox/etc/CLANG_TI.4.0.1.cmake

      - name: Set up Python
        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install pip dependencies
        run: |
            pip install --upgrade pip
            pip install pyyaml

      - name: Generate manifest file
        shell: bash
        run: |
          # Get the toolbox version
          TOOLBOX_VERSION=$(git describe --tags || echo "untagged")

          # Set binary extension (default to empty string if not set)
          BINARY_EXTENSION=${{ matrix.binary_extension }}
          [ -z "$BINARY_EXTENSION" ] && BINARY_EXTENSION=""

          # Get absolute path to the toolbox directory
          TOOLBOX_ABS_PATH=$(pwd)/cmsis-toolbox

          # Get host machine configuration
          HOST=${{ matrix.target }}-${{ matrix.arch }}

          # Run the Python script to generate the manifest file
          python ./scripts/generate_manifest.py \
            -d "$TOOLBOX_ABS_PATH" \
            -v "$TOOLBOX_VERSION" \
            -e "$BINARY_EXTENSION" \
            --host "$HOST"

      - name: Archive cmsis-toolbox
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: cmsis-toolbox-${{ matrix.target }}-${{ matrix.arch }}
          path: cmsis-toolbox
          retention-days: ${{ env.retention-days }}

      # Debian Package
      - name: Update dependencies
        if: ${{ matrix.target == 'linux' && matrix.arch == 'amd64' }}
        run: |
            sudo apt-get update
            sudo apt-get install ninja-build

      - name: Create Debian Package
        if: ${{ matrix.target == 'linux' && matrix.arch == 'amd64' }}
        run: |
            git fetch
            mkdir -p ./debian/build
            cmake -G Ninja -S ./debian -B ./debian/build -DTOOLBOX_ROOT=$(realpath ./cmsis-toolbox) -DTOOLBOX_VERSION=$(git describe --tags --abbrev=0)+nightly-$(date +%Y%m%d%H%M)
            cd ./debian/build
            cpack -G DEB

      - name: Archive Debian Package
        if: ${{ matrix.target == 'linux' && matrix.arch == 'amd64' }}
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: cmsis-toolbox-debian-package
          path: ./debian/build/cmsis-toolbox_*_amd64.deb
          retention-days: ${{ env.retention-days }}

  run-tests:
    name: Run Tests [${{ matrix.target }}-${{ matrix.arch }}]
    needs: [ matrix_prep, create-toolbox ]
    runs-on: ${{ matrix.runs_on }}
    continue-on-error: true
    strategy:
      matrix: ${{ fromJson(needs.matrix_prep.outputs.test) }}

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
        with:
          egress-policy: audit

      - name: Checkout toolbox repository
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

      - name: Set up Python
        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install pip dependencies
        run: |
            pip install --upgrade pip
            pip install -r test/e2e/requirements.txt

      - name: Setup vcpkg environment
        uses: ARM-software/cmsis-actions/vcpkg@afc8e1a46fad8a5e1a08f8477b71050d442f60a7 # v1
        with:
          config: "./test/vcpkg-configuration.json"
          vcpkg-downloads: "${{ github.workspace }}/.vcpkg/downloads"
          cache: "-"

      - name: Activate Arm tool license
        run: |
          armlm activate --server https://mdk-preview.keil.arm.com --product KEMDK-COM0
        working-directory: ./test

      - name: Download cmsis toolbox ${{ matrix.target }} ${{ matrix.arch }}
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: cmsis-toolbox-${{ matrix.target }}-${{ matrix.arch }}
          path: cmsis-toolbox

      - name: Set execution permissions
        run: |
          chmod -R +x ./cmsis-toolbox/bin/*

      - name: Get toolbox path
        if: always()
        id: toolbox_path
        shell: bash
        run: |
          echo "path=$(echo "$(pwd)/cmsis-toolbox/bin")" >> $GITHUB_OUTPUT

      - name: Run Test
        id: run_robot
        shell: bash
        continue-on-error: true
        working-directory: ./test
        env:
          IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_TOKEN }}
        run: |
          export PATH="${{steps.toolbox_path.outputs.path}}:$PATH"
          robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
            --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \
            --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \
            --name ${{ matrix.target }}-${{ matrix.arch }} \
            ./e2e
          echo "exit_code=$?" >> $GITHUB_OUTPUT

      - name: Archieve test results
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: reports-${{ matrix.target }}-${{ matrix.arch }}
          path: ./test/reports-${{ matrix.target }}-${{ matrix.arch }}
          retention-days: ${{ env.retention-days }}

      - name: Fail job if Robot tests failed
        if: steps.run_robot.outputs.exit_code != '0'
        run: |
          echo "Robot tests failed"
          exit 1

  consolidate-report:
    needs: [run-tests]
    runs-on: ubuntu-latest
    permissions: write-all
    if: needs.run-tests.result == 'success' || needs.run-tests.result == 'failure'
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
      with:
        egress-policy: audit

    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

    - name: Set up Python
      uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
      with:
        python-version: '3.11'
        cache: 'pip'

    - name: Install pip dependencies
      run: |
          pip install --upgrade pip
          pip install -r test/e2e/requirements.txt

    - name: Download reports
      uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
      with:
        path: artifacts
        pattern: reports-*

    - name: Consolidate robot test results
      working-directory: artifacts
      continue-on-error: true
      run: |
        python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
          ./reports-windows-amd64/output.xml \
          ./reports-linux-amd64/output.xml

    - name: Generate Summary report
      if: always()
      run: |
        python ./test/e2e/lib/execution_summary.py artifacts \
          -r ./test/e2e/reference.md \
          -o artifacts/collective_robot_results/output.xml \
          -m summary_report.md

    - name: Print E2E Report
      if: always()
      run: cat summary_report.md >> $GITHUB_STEP_SUMMARY

    - name: Archive consolidated test results
      if: always()
      uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
      with:
        name: consolidated-reports
        path: artifacts/collective_robot_results
        retention-days: ${{ env.retention-days }}

  performance-check:
    needs: [ matrix_prep, create-toolbox ]
    runs-on: [ ubuntu-latest ]
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
        with:
          egress-policy: audit

      - name: Checkout toolbox repository
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

      - name: Set up Python
        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install pip dependencies
        run: |
            pip install --upgrade pip
            pip install -r test/performance/requirements.txt

      - name: Setup vcpkg environment
        uses: ARM-software/cmsis-actions/vcpkg@afc8e1a46fad8a5e1a08f8477b71050d442f60a7 # v1
        with:
          config: "./test/vcpkg-configuration.json"
          vcpkg-downloads: "${{ github.workspace }}/.vcpkg/downloads"
          cache: "-"

      - name: Activate Arm tool license
        run: |
          armlm activate --server https://mdk-preview.keil.arm.com --product KEMDK-COM0
        working-directory: ./test

      - name: Install Dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y hyperfine unzip

      # Download the reference toolbox release
      # This needed to be updated when a positive performance delta is expected
      - name: Download Reference cmsis-toolbox
        continue-on-error: true
        uses: robinraju/release-downloader@daf26c55d821e836577a15f77d86ddc078948b05 # v1.12
        with:
          repository: Open-CMSIS-Pack/cmsis-toolbox
          tag: "2.12.0-test-reference"
          fileName: "cmsis-toolbox-linux-amd64.zip"

      - name: Unzip Reference cmsis-toolbox
        shell: bash
        run: |
          mkdir -p cmsis-toolbox-reference
          unzip cmsis-toolbox-linux-amd64.zip -d cmsis-toolbox-reference

      - name: Set Execution Permissions
        run: |
          chmod -R +x ./cmsis-toolbox-reference/bin/*

      - name: Get Reference Toolbox Path
        if: always()
        id: ref_toolbox_path
        shell: bash
        run: |
          echo "path=$(echo "$(pwd)/cmsis-toolbox-reference/bin")" >> $GITHUB_OUTPUT

      - name: Print Ref Binary Versions
        if: always()
        shell: bash
        run: |
          export PATH="${{steps.ref_toolbox_path.outputs.path}}:$PATH"
          csolution -V
          cpackget -V
          cbuild -V
          cbuild2cmake -V
          cbuildgen -V

      - name: Clone TestData Repositories
        shell: bash
        working-directory: ./test
        run: |
          set -x  # Enable debugging
          mkdir -p ref_test_dir
          cat performance/input_example_repos.txt | tr -d '\r' > repos_clean.txt  # Ensure no Windows line endings
          while IFS= read -r repo || [[ -n "$repo" ]]; do
            if [[ -n "$repo" ]]; then
              echo "Cloning $repo..."
              git clone "$repo" "ref_test_dir/$(basename "$repo" .git)" || echo "Failed to clone $repo"
            fi
          done < repos_clean.txt

      - name: Collect Performance Metrics (Reference Toolbox)
        working-directory: ./test
        run: |
          export PATH="${{steps.ref_toolbox_path.outputs.path}}:$PATH"
          hyperfine --warmup 3 --runs 5 \
            "cbuild setup ./ref_test_dir/csolution-examples/Hello/Hello.csolution.yml -p -S --update-rte --toolchain AC6" \
            "cbuild setup ./ref_test_dir/vscode-get-started/get_started.csolution.yml -p -S --update-rte --toolchain AC6" \
            "cbuild setup ./ref_test_dir/CMSIS-RTX/Examples/Blinky/Blinky.csolution.yml -p -S --update-rte --toolchain AC6" \
            "cbuild setup ./ref_test_dir/CMSIS-RTX/Examples/Blinky/Blinky.csolution.yml -p -S --update-rte --toolchain GCC" \
            "cbuild setup ./ref_test_dir/CMSIS-RTX/Examples/Blinky/Blinky.csolution.yml -p -S --update-rte --toolchain CLANG" \
            --export-markdown ref_benchmark_results.md \
            --export-json ref_benchmark_results.json \
            --show-output
          # Remove the reference toolbox after the benchmark
          rm -rf ${{steps.ref_toolbox_path.outputs.path}}

      - name: Download cmsis-toolbox
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: cmsis-toolbox-linux-amd64
          path: cmsis-toolbox

      - name: Set Execution Permissions
        run: |
          chmod -R +x ./cmsis-toolbox/bin/*

      - name: Get Toolbox Path
        if: always()
        id: toolbox_path
        shell: bash
        run: |
          echo "path=$(echo "$(pwd)/cmsis-toolbox/bin")" >> $GITHUB_OUTPUT

      - name: Print Curr Binary Versions
        if: always()
        shell: bash
        run: |
          export PATH="${{steps.toolbox_path.outputs.path}}:$PATH"
          export CMSIS_PACK_ROOT="$(pwd)/local_pack_root"
          csolution -V
          cpackget -V
          cbuild -V
          cbuild2cmake -V
          cbuildgen -V

      - name: Clone Repositories
        shell: bash
        working-directory: ./test
        run: |
          set -x  # Enable debugging
          mkdir -p curr_test_dir
          cat performance/input_example_repos.txt | tr -d '\r' > repos_clean.txt  # Ensure no Windows line endings
          while IFS= read -r repo || [[ -n "$repo" ]]; do
            if [[ -n "$repo" ]]; then
              echo "Cloning $repo..."
              git clone "$repo" "curr_test_dir/$(basename "$repo" .git)" || echo "Failed to clone $repo"
            fi
          done < repos_clean.txt

      - name: Collect Performance Metrics (Latest Toolbox)
        working-directory: ./test
        run: |
          export PATH="${{steps.toolbox_path.outputs.path}}:$PATH"
          export CMSIS_PACK_ROOT="$(pwd)/local_pack_root"
          hyperfine --warmup 3 --runs 5 \
            "cbuild setup ./curr_test_dir/csolution-examples/Hello/Hello.csolution.yml -p -S --update-rte --toolchain AC6" \
            "cbuild setup ./curr_test_dir/vscode-get-started/get_started.csolution.yml -p -S --update-rte --toolchain AC6" \
            "cbuild setup ./curr_test_dir/CMSIS-RTX/Examples/Blinky/Blinky.csolution.yml -p -S --update-rte --toolchain AC6" \
            "cbuild setup ./curr_test_dir/CMSIS-RTX/Examples/Blinky/Blinky.csolution.yml -p -S --update-rte --toolchain GCC" \
            "cbuild setup ./curr_test_dir/CMSIS-RTX/Examples/Blinky/Blinky.csolution.yml -p -S --update-rte --toolchain CLANG" \
            --export-markdown curr_benchmark_results.md \
            --export-json curr_benchmark_results.json \
            --show-output

      - name: Generate Comparison Report
        if: always()
        working-directory: ./test
        run: |
          # Generate Comparison report
          # Path to the reference benchmark results (from last known reference)
          # Path to the current benchmark results (from the latest run)
          # --verbose: Verbose output
          # --output: Path to the output comparison report
          python ./performance/generate_comparison.py \
            ref_benchmark_results.md \
            curr_benchmark_results.md \
            --verbose \
            --output comparison_results.md
          cat comparison_results.md >> $GITHUB_STEP_SUMMARY

      - name: Compare Benchmark Results
        working-directory: ./test
        run: |
          # Run the performance regression check script
          # -r: Path to the reference benchmark results (from last known reference)
          # -c: Path to the current benchmark results (from the latest run)
          # -p: Permissible limit multiplier (e.g., 1.10 allows up to 10% permissible degrade)
          python ./performance/check_perf_regression.py \
            -r ref_benchmark_results.json \
            -c curr_benchmark_results.json \
            -p 1.10