Skip to content

Commit 4c1d69c

Browse files
soumeh01soumeh01
committed
Update test reference
1 parent e44cadc commit 4c1d69c

File tree

1 file changed

+150
-150
lines changed

1 file changed

+150
-150
lines changed

‎.github/workflows/nightly.yml‎

Lines changed: 150 additions & 150 deletions
Original file line numberDiff line numberDiff line change
@@ -351,150 +351,150 @@ jobs:
351351
path: ./debian/build/cmsis-toolbox_*_amd64.deb
352352
retention-days: ${{ env.retention-days }}
353353

354-
run-tests:
355-
name: Run Tests [${{ matrix.target }}-${{ matrix.arch }}]
356-
needs: [ matrix_prep, create-toolbox ]
357-
runs-on: ${{ matrix.runs_on }}
358-
continue-on-error: true
359-
strategy:
360-
matrix: ${{ fromJson(needs.matrix_prep.outputs.test) }}
361-
362-
steps:
363-
- name: Harden Runner
364-
uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
365-
with:
366-
egress-policy: audit
367-
368-
- name: Checkout toolbox repository
369-
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
370-
371-
- name: Set up Python
372-
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
373-
with:
374-
python-version: '3.11'
375-
cache: 'pip'
376-
377-
- name: Install pip dependencies
378-
run: |
379-
pip install --upgrade pip
380-
pip install -r test/e2e/requirements.txt
381-
382-
- name: Setup vcpkg environment
383-
uses: ARM-software/cmsis-actions/vcpkg@afc8e1a46fad8a5e1a08f8477b71050d442f60a7 # v1
384-
with:
385-
config: "./test/vcpkg-configuration.json"
386-
vcpkg-downloads: "${{ github.workspace }}/.vcpkg/downloads"
387-
cache: "-"
388-
389-
- name: Activate Arm tool license
390-
run: |
391-
armlm activate --server https://mdk-preview.keil.arm.com --product KEMDK-COM0
392-
working-directory: ./test
393-
394-
- name: Download cmsis toolbox ${{ matrix.target }} ${{ matrix.arch }}
395-
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
396-
with:
397-
name: cmsis-toolbox-${{ matrix.target }}-${{ matrix.arch }}
398-
path: cmsis-toolbox
399-
400-
- name: Set execution permissions
401-
run: |
402-
chmod -R +x ./cmsis-toolbox/bin/*
403-
404-
- name: Get toolbox path
405-
if: always()
406-
id: toolbox_path
407-
shell: bash
408-
run: |
409-
echo "path=$(echo "$(pwd)/cmsis-toolbox/bin")" >> $GITHUB_OUTPUT
410-
411-
- name: Run Test
412-
id: run_robot
413-
shell: bash
414-
continue-on-error: true
415-
working-directory: ./test
416-
env:
417-
IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_TOKEN }}
418-
run: |
419-
export PATH="${{steps.toolbox_path.outputs.path}}:$PATH"
420-
robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
421-
--variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \
422-
--consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \
423-
--name ${{ matrix.target }}-${{ matrix.arch }} \
424-
./e2e
425-
echo "exit_code=$?" >> $GITHUB_OUTPUT
426-
427-
- name: Archieve test results
428-
if: always()
429-
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
430-
with:
431-
name: reports-${{ matrix.target }}-${{ matrix.arch }}
432-
path: ./test/reports-${{ matrix.target }}-${{ matrix.arch }}
433-
retention-days: ${{ env.retention-days }}
434-
435-
- name: Fail job if Robot tests failed
436-
if: steps.run_robot.outputs.exit_code != '0'
437-
run: |
438-
echo "Robot tests failed"
439-
exit 1
440-
441-
consolidate-report:
442-
needs: [run-tests]
443-
runs-on: ubuntu-latest
444-
permissions: write-all
445-
if: needs.run-tests.result == 'success' || needs.run-tests.result == 'failure'
446-
steps:
447-
- name: Harden Runner
448-
uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
449-
with:
450-
egress-policy: audit
451-
452-
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
453-
454-
- name: Set up Python
455-
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
456-
with:
457-
python-version: '3.11'
458-
cache: 'pip'
459-
460-
- name: Install pip dependencies
461-
run: |
462-
pip install --upgrade pip
463-
pip install -r test/e2e/requirements.txt
464-
465-
- name: Download reports
466-
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
467-
with:
468-
path: artifacts
469-
pattern: reports-*
470-
471-
- name: Consolidate robot test results
472-
working-directory: artifacts
473-
continue-on-error: true
474-
run: |
475-
python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
476-
./reports-windows-amd64/output.xml \
477-
./reports-linux-amd64/output.xml
478-
479-
- name: Generate Summary report
480-
if: always()
481-
run: |
482-
python ./test/e2e/lib/execution_summary.py artifacts \
483-
-r ./test/e2e/reference.md \
484-
-o artifacts/collective_robot_results/output.xml \
485-
-m summary_report.md
486-
487-
- name: Print E2E Report
488-
if: always()
489-
run: cat summary_report.md >> $GITHUB_STEP_SUMMARY
490-
491-
- name: Archive consolidated test results
492-
if: always()
493-
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
494-
with:
495-
name: consolidated-reports
496-
path: artifacts/collective_robot_results
497-
retention-days: ${{ env.retention-days }}
354+
# run-tests:
355+
# name: Run Tests [${{ matrix.target }}-${{ matrix.arch }}]
356+
# needs: [ matrix_prep, create-toolbox ]
357+
# runs-on: ${{ matrix.runs_on }}
358+
# continue-on-error: true
359+
# strategy:
360+
# matrix: ${{ fromJson(needs.matrix_prep.outputs.test) }}
361+
362+
# steps:
363+
# - name: Harden Runner
364+
# uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
365+
# with:
366+
# egress-policy: audit
367+
368+
# - name: Checkout toolbox repository
369+
# uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
370+
371+
# - name: Set up Python
372+
# uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
373+
# with:
374+
# python-version: '3.11'
375+
# cache: 'pip'
376+
377+
# - name: Install pip dependencies
378+
# run: |
379+
# pip install --upgrade pip
380+
# pip install -r test/e2e/requirements.txt
381+
382+
# - name: Setup vcpkg environment
383+
# uses: ARM-software/cmsis-actions/vcpkg@afc8e1a46fad8a5e1a08f8477b71050d442f60a7 # v1
384+
# with:
385+
# config: "./test/vcpkg-configuration.json"
386+
# vcpkg-downloads: "${{ github.workspace }}/.vcpkg/downloads"
387+
# cache: "-"
388+
389+
# - name: Activate Arm tool license
390+
# run: |
391+
# armlm activate --server https://mdk-preview.keil.arm.com --product KEMDK-COM0
392+
# working-directory: ./test
393+
394+
# - name: Download cmsis toolbox ${{ matrix.target }} ${{ matrix.arch }}
395+
# uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
396+
# with:
397+
# name: cmsis-toolbox-${{ matrix.target }}-${{ matrix.arch }}
398+
# path: cmsis-toolbox
399+
400+
# - name: Set execution permissions
401+
# run: |
402+
# chmod -R +x ./cmsis-toolbox/bin/*
403+
404+
# - name: Get toolbox path
405+
# if: always()
406+
# id: toolbox_path
407+
# shell: bash
408+
# run: |
409+
# echo "path=$(echo "$(pwd)/cmsis-toolbox/bin")" >> $GITHUB_OUTPUT
410+
411+
# - name: Run Test
412+
# id: run_robot
413+
# shell: bash
414+
# continue-on-error: true
415+
# working-directory: ./test
416+
# env:
417+
# IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_TOKEN }}
418+
# run: |
419+
# export PATH="${{steps.toolbox_path.outputs.path}}:$PATH"
420+
# robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
421+
# --variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \
422+
# --consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \
423+
# --name ${{ matrix.target }}-${{ matrix.arch }} \
424+
# ./e2e
425+
# echo "exit_code=$?" >> $GITHUB_OUTPUT
426+
427+
# - name: Archieve test results
428+
# if: always()
429+
# uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
430+
# with:
431+
# name: reports-${{ matrix.target }}-${{ matrix.arch }}
432+
# path: ./test/reports-${{ matrix.target }}-${{ matrix.arch }}
433+
# retention-days: ${{ env.retention-days }}
434+
435+
# - name: Fail job if Robot tests failed
436+
# if: steps.run_robot.outputs.exit_code != '0'
437+
# run: |
438+
# echo "Robot tests failed"
439+
# exit 1
440+
441+
# consolidate-report:
442+
# needs: [run-tests]
443+
# runs-on: ubuntu-latest
444+
# permissions: write-all
445+
# if: needs.run-tests.result == 'success' || needs.run-tests.result == 'failure'
446+
# steps:
447+
# - name: Harden Runner
448+
# uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
449+
# with:
450+
# egress-policy: audit
451+
452+
# - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
453+
454+
# - name: Set up Python
455+
# uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
456+
# with:
457+
# python-version: '3.11'
458+
# cache: 'pip'
459+
460+
# - name: Install pip dependencies
461+
# run: |
462+
# pip install --upgrade pip
463+
# pip install -r test/e2e/requirements.txt
464+
465+
# - name: Download reports
466+
# uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
467+
# with:
468+
# path: artifacts
469+
# pattern: reports-*
470+
471+
# - name: Consolidate robot test results
472+
# working-directory: artifacts
473+
# continue-on-error: true
474+
# run: |
475+
# python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
476+
# ./reports-windows-amd64/output.xml \
477+
# ./reports-linux-amd64/output.xml
478+
479+
# - name: Generate Summary report
480+
# if: always()
481+
# run: |
482+
# python ./test/e2e/lib/execution_summary.py artifacts \
483+
# -r ./test/e2e/reference.md \
484+
# -o artifacts/collective_robot_results/output.xml \
485+
# -m summary_report.md
486+
487+
# - name: Print E2E Report
488+
# if: always()
489+
# run: cat summary_report.md >> $GITHUB_STEP_SUMMARY
490+
491+
# - name: Archive consolidated test results
492+
# if: always()
493+
# uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
494+
# with:
495+
# name: consolidated-reports
496+
# path: artifacts/collective_robot_results
497+
# retention-days: ${{ env.retention-days }}
498498

499499
performance-check:
500500
needs: [ matrix_prep, create-toolbox ]
@@ -540,17 +540,17 @@ jobs:
540540
# This needed to be updated when a positive performance delta is expected
541541
- name: Download Reference cmsis-toolbox
542542
continue-on-error: true
543-
uses: robinraju/[email protected]
543+
uses: robinraju/release-downloader@daf26c55d821e836577a15f77d86ddc078948b05 # v1.12
544544
with:
545545
repository: Open-CMSIS-Pack/cmsis-toolbox
546-
tag: "2.12.0"
547-
fileName: "cmsis-toolbox-linux-amd64.tar.gz"
546+
tag: "2.11.0-test-reference"
547+
fileName: "cmsis-toolbox-2.12-linux-amd64.zip"
548548

549549
- name: Unzip Reference cmsis-toolbox
550550
shell: bash
551551
run: |
552-
tar -xvf cmsis-toolbox-linux-amd64.tar.gz
553-
mv cmsis-toolbox-linux-amd64 cmsis-toolbox-reference
552+
mkdir -p cmsis-toolbox-reference
553+
unzip cmsis-toolbox-2.12-linux-amd64.zip -d cmsis-toolbox-reference
554554
555555
- name: Set Execution Permissions
556556
run: |
@@ -688,4 +688,4 @@ jobs:
688688
python ./performance/check_perf_regression.py \
689689
-r ref_benchmark_results.json \
690690
-c curr_benchmark_results.json \
691-
-p 1.20
691+
-p 1.10

0 commit comments

Comments
 (0)