ci on 309 #750
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Package, test and upload core | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| env: | |
| BRANCH_NAME: origin/${{ github.head_ref || github.ref_name }} | |
| jobs: | |
| build-env: | |
| name: Prepare build environment | |
| runs-on: ubuntu-latest | |
| outputs: | |
| CORE_TAG: ${{ env.CORE_TAG }} | |
| CORE_HASH: ${{ env.CORE_HASH }} | |
| ALL_BOARD_DATA: ${{ env.ALL_BOARD_DATA }} | |
| ALL_BOARD_FQBNS: ${{ env.ALL_BOARD_FQBNS }} | |
| ARTIFACTS: ${{ env.ARTIFACTS }} | |
| SUB_ARCHES: ${{ env.SUB_ARCHES }} | |
| steps: | |
| - name: Install OS dependencies | |
| working-directory: /opt | |
| run: | | |
| sudo apt-get remove --purge man-db -y # skips the mandb triggers | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends git cmake wget python3-pip ninja-build | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| fetch-tags: true | |
| - name: Initialize Zephyr environment | |
| run: | | |
| yes | ./extra/bootstrap.sh -o=--filter=tree:0 | |
| echo "CORE_HASH=$(git describe --always $BRANCH_NAME)" >> "$GITHUB_ENV" | |
| echo "ALL_BOARD_DATA=$(extra/get_board_details.sh | jq -c 'sort_by(.variant)')" >> "$GITHUB_ENV" | |
| echo "## Building \`$(extra/get_core_version.sh)\`" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Map output packages | |
| # needs the above env vars to be usable | |
| run: | | |
| echo "CORE_TAG=$(git describe --tags --exact-match $BRANCH_NAME 2>/dev/null || echo $CORE_HASH)" >> "$GITHUB_ENV" | |
| echo "ALL_BOARD_FQBNS=$(jq -c 'map((. + {link_mode: "static"}), (. + {link_mode: "dynamic"}))' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" | |
| echo "ARTIFACTS=$(jq -c '["zephyr"] + (map(.artifact) | unique)' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" | |
| echo "SUB_ARCHES=$(jq -c 'map(.subarch) | unique' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV" | |
| (cd && tar cphf - .cmake work zephyr-sdk-* | zstd > build-env.tar.zstd) | |
| tar cphf - cores/arduino/api | zstd > arduino-api.tar.zstd | |
| - name: Archive build environment | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-env | |
| path: ~/build-env.tar.zstd | |
| - name: Archive API snapshot | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: arduino-api | |
| path: arduino-api.tar.zstd | |
| build-board: | |
| # NOTE: this name is hardcoded in ci_inspect_logs.py | |
| name: Build for ${{ matrix.board }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-env | |
| env: | |
| CCACHE_IGNOREOPTIONS: -specs=* | |
| ARTIFACT_TAG: ${{ needs.build-env.outputs.CORE_HASH }}-${{ matrix.board }} | |
| REPORT: reports/zephyr-${{ matrix.variant }} | |
| strategy: | |
| matrix: | |
| include: | |
| ${{ fromJSON( needs.build-env.outputs.ALL_BOARD_DATA ) }} | |
| fail-fast: false | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: /home/runner | |
| name: build-env | |
| - name: Restore build environment | |
| run: | | |
| sudo apt-get remove --purge man-db -y # skips the mandb triggers | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends git cmake wget python3-pip ninja-build ccache | |
| (cd ~ && tar --use-compress-program=unzstd -xpf build-env.tar.zstd && rm build-env.tar.zstd) | |
| - name: ccache | |
| uses: hendrikmuhs/ccache-action@v1.2 | |
| with: | |
| verbose: 1 | |
| key: ${{ github.job }}-${{ matrix.board }} | |
| - name: Build loader | |
| shell: bash | |
| run: | | |
| mkdir -p reports | |
| if ! ./extra/build.sh ${{ matrix.board }} 1> >(tee $REPORT.stdout) 2> >(tee $REPORT.stderr) ; then | |
| echo "### :x: ${{ matrix.board }} (\`${{ matrix.variant }}\`) build errors" > $GITHUB_STEP_SUMMARY | |
| echo >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| cat $REPORT.stderr >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| fi | |
| # look for warnings (errors are a happy path!) | |
| grep -i "warning:" $REPORT.stdout > $REPORT.warnings || true | |
| # extract the memory usage table (from the header to the first non-% line) | |
| # override the size of the Flash with the size of the loader partition | |
| # and add the size of the sketch partition (not reported by Zephyr) | |
| LOADER_SIZE=$(( $(cat variants/${{ matrix.variant }}/syms-static.ld | grep '_loader_max_size' | cut -d '=' -f 2 | tr -d ') ;') )) | |
| SKETCH_SIZE=$(( $(cat variants/${{ matrix.variant }}/syms-static.ld | grep '_sketch_max_size' | cut -d '=' -f 2 | tr -d ') ;') )) | |
| cat $REPORT.stdout | sed -n '/^Memory region/,/^[^%]*$/p' | head -n -1 \ | |
| | awk 'BEGIN {split("B KB MB GB", u); for(i in u) m[u[i]]=1024^(i-1)} /:/ {print "[\"" $1 "\"," $2*m[$3] "," $4*m[$5] "]"}' \ | |
| | sort \ | |
| | jq -s "map((select(.[0] == \"FLASH:\") | .[2]) |= ${LOADER_SIZE}) + [ [ \"SKETCH:\", 0, ${SKETCH_SIZE} ] ]" > $REPORT.meminfo | |
| jq | |
| - name: Package board artifacts | |
| if: ${{ !cancelled() }} | |
| run: | | |
| cp firmwares/zephyr-${{ matrix.variant }}.config $REPORT.config | |
| echo "REPORT_FILES<<EOF" >> $GITHUB_ENV | |
| ls reports/* >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| # Archive built binaries (and build dir on failure) | |
| [ "${{ job.status }}" == "failure" ] && FULL_BUILD_DIR="build/${{ matrix.variant }}/" | |
| tar chf - \ | |
| firmwares/*${{ matrix.variant }}* \ | |
| variants/${{ matrix.variant }}/ \ | |
| ${FULL_BUILD_DIR} \ | |
| | zstd > binaries-${ARTIFACT_TAG}.tar.zstd | |
| - name: Archive board binaries | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ format('{0}binaries-{1}', (job.status == 'failure') && 'failed-' || '', env.ARTIFACT_TAG) }} | |
| path: binaries-${{ env.ARTIFACT_TAG }}.tar.zstd | |
| - name: Archive build reports | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-report-${{ env.ARTIFACT_TAG }} | |
| path: ${{ env.REPORT_FILES }} | |
| package-core: | |
| name: Package ${{ matrix.artifact }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-env | |
| - build-board | |
| env: | |
| ALL_BOARD_DATA: ${{ needs.build-env.outputs.ALL_BOARD_DATA }} | |
| CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }} | |
| CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }} | |
| strategy: | |
| matrix: | |
| artifact: ${{ fromJSON( needs.build-env.outputs.ARTIFACTS ) }} | |
| fail-fast: false | |
| if: ${{ !cancelled() && needs.build-env.result == 'success' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| fetch-tags: true | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: . | |
| name: arduino-api | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: . | |
| pattern: binaries-* | |
| merge-multiple: true | |
| - name: Package core | |
| run: | | |
| rm -f cores/arduino/api # remove broken symlink | |
| tar --use-compress-program=unzstd -xpf arduino-api.tar.zstd | |
| for f in binaries-*.tar.zstd ; do | |
| tar --use-compress-program=unzstd -xpf $f | |
| done | |
| ./extra/package_core.sh ${{ matrix.artifact }} ${CORE_TAG} distrib/${CORE_ARTIFACT}.tar.bz2 | |
| - uses: actions/upload-artifact@v4 | |
| if: ${{ success() || failure() }} | |
| with: | |
| name: ${{ env.CORE_ARTIFACT }} | |
| path: distrib/${{ env.CORE_ARTIFACT }}.tar.bz2 | |
| cleanup-build: | |
| name: Clean up intermediates | |
| runs-on: ubuntu-latest | |
| needs: | |
| - package-core | |
| steps: | |
| - uses: geekyeggo/delete-artifact@v5.1.0 | |
| with: | |
| name: | | |
| arduino-api | |
| binaries-* | |
| build-env | |
| failOnError: false | |
| test-core: | |
| # NOTE: this name is hardcoded in ci_inspect_logs.py | |
| name: Test ${{ matrix.board }}:${{ matrix.link_mode }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-env | |
| - package-core | |
| strategy: | |
| matrix: | |
| include: | |
| ${{ fromJSON( needs.build-env.outputs.ALL_BOARD_FQBNS ) }} | |
| fail-fast: false | |
| env: | |
| PLAT: arduino:${{ matrix.subarch }} | |
| FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }}:link_mode=${{ matrix.link_mode }} | |
| CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.build-env.outputs.CORE_HASH }} | |
| ARTIFACT_TAG: ${{ needs.build-env.outputs.CORE_HASH }}-${{ matrix.board }}-${{ matrix.link_mode }} | |
| if: ${{ !cancelled() && needs.build-env.result == 'success' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| sparse-checkout: | | |
| extra/ci_test_list.sh | |
| extra/artifacts/ | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.CORE_ARTIFACT }} | |
| - name: Set up core | |
| run: | | |
| tar xf ${CORE_ARTIFACT}.tar.bz2 # will create ArduinoCore-zephyr/ | |
| echo "REPORT_FILE=$(echo ${FQBN} | tr ':' '-').json" >> $GITHUB_ENV | |
| - name: Get test sketches | |
| run: | | |
| # sets ALL_TESTS and ALL_LIBRARIES env vars | |
| extra/ci_test_list.sh ${{ matrix.artifact }} ${{ matrix.variant }} | |
| - name: Compile tests for ${{ matrix.board }} | |
| uses: pillo79/compile-sketches@4a1dead03155c17ddedc373699d7aa80a78a6c7d # next | |
| with: | |
| fqbn: ${{ env.FQBN }} | |
| platforms: | | |
| # Use Board Manager version first, to install the toolchain | |
| - name: ${{ env.PLAT }} | |
| - name: ${{ env.PLAT }} | |
| source-path: "ArduinoCore-zephyr" | |
| sketch-paths: | | |
| ${{ env.ALL_TESTS }} | |
| libraries: | | |
| ${{ env.ALL_LIBRARIES }} | |
| cli-compile-flags: | | |
| - '--build-property' | |
| - 'compiler.c.extra_flags=-Wno-type-limits -Wno-missing-field-initializers' | |
| - '--build-property' | |
| - 'compiler.cpp.extra_flags=-Wno-type-limits -Wno-missing-field-initializers' | |
| verbose: 'false' | |
| enable-deltas-report: 'false' | |
| enable-issues-report: 'true' | |
| always-succeed: 'true' | |
| - name: Prepare log | |
| if: ${{ success() || failure() }} | |
| run: | | |
| [ ! -f sketches-reports/${REPORT_FILE} ] && mkdir -p sketches-reports && echo "{}" > sketches-reports/${REPORT_FILE} | |
| sed -i -e 's!/home/runner/.arduino15/packages/arduino/hardware/zephyr/[^/]*/!!g' sketches-reports/${REPORT_FILE} | |
| - uses: actions/upload-artifact@v4 | |
| if: ${{ success() || failure() }} | |
| with: | |
| name: test-report-${{ env.ARTIFACT_TAG }} | |
| path: sketches-reports/* | |
| inspect-logs: | |
| name: Analyze logs | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-env | |
| - package-core | |
| - test-core | |
| if: ${{ !cancelled() && needs.build-env.result == 'success' }} | |
| env: | |
| ALL_BOARD_DATA: ${{ needs.build-env.outputs.ALL_BOARD_DATA }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GH_REPO: ${{ github.repository }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| fetch-tags: true | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: . | |
| pattern: "*-report-*" | |
| merge-multiple: true | |
| - run: | | |
| export WORKFLOW_JOBS=$(gh run view ${{ github.run_id }} --attempt ${{ github.run_attempt }} --json jobs --jq '.jobs') | |
| extra/ci_inspect_logs.py result summary full_log | |
| cat summary >> $GITHUB_STEP_SUMMARY | |
| cat full_log >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| mkdir -p comment-request | |
| echo "${{ github.event.pull_request.number }}" > comment-request/pr_number | |
| echo -e "## Built \`$(extra/get_core_version.sh)\`\n" > comment-request/comment_body | |
| echo >> comment-request/comment_body | |
| cat summary | sed -e 's!\${\\color{\S*}\(.*\)}\$!\1!g' -e 's!\\%!%!g' >> comment-request/comment_body | |
| fi | |
| - name: Archive comment information | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ github.event_name == 'pull_request' }} | |
| with: | |
| name: comment-request | |
| path: comment-request/ | |
| retention-days: 1 | |
| - name: Clean up intermediate artifacts | |
| uses: geekyeggo/delete-artifact@v5.1.0 | |
| with: | |
| name: | | |
| build-report-* | |
| test-report-* | |
| failOnError: false | |
| - run: | | |
| [ $(cat result) == 'PASSED') ] && exit 0 # otherwise fail | |
| verify-core: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-env | |
| - package-core | |
| - inspect-logs | |
| if: ${{ !cancelled() }} | |
| steps: | |
| - name: CI run result | |
| run: | | |
| exit ${{ contains(needs.*.result, 'failure') && '1' || '0' }} | |
| publish-core: | |
| name: Publish core | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }} | |
| needs: | |
| - build-env | |
| - package-core | |
| - inspect-logs | |
| environment: production | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: . | |
| pattern: ArduinoCore-* | |
| merge-multiple: true | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ secrets.IAM_ROLE }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Upload artifact | |
| run: | | |
| for f in ArduinoCore-*.tar.bz2 ; do | |
| aws s3 cp $f s3://${{ secrets.S3_BUCKET }}/ | |
| done | |
| prepare-json: | |
| name: Prepare jsons | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-env | |
| - package-core | |
| - inspect-logs | |
| env: | |
| CORE_TAG: ${{ needs.build-env.outputs.CORE_TAG }} | |
| CORE_HASH: ${{ needs.build-env.outputs.CORE_HASH }} | |
| ARTIFACTS: ${{ needs.build-env.outputs.ARTIFACTS }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| fetch-tags: true | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: . | |
| pattern: ArduinoCore-* | |
| merge-multiple: true | |
| - name: Prepare package index snippets | |
| run: | | |
| jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do | |
| ARTIFACT_FILE=ArduinoCore-${artifact}-${CORE_HASH}.tar.bz2 | |
| PACKAGE_JSON=ArduinoCore-${artifact}-${CORE_TAG}.json | |
| ./extra/gen_package_index_json.sh ${artifact} ${ARTIFACT_FILE} ${PACKAGE_JSON} | |
| done | |
| - name: Archive package index snippets | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ArduinoCore-zephyr-${{ env.CORE_TAG }}-jsons | |
| path: ArduinoCore-*-${{ env.CORE_TAG }}.json |