Skip to content

Commit 2cc7acf

Browse files
soumeh01soumeh01
authored
Run E2E with CMake 4.2.1 (#487)
## Changes <!-- List the changes this PR introduces --> - Run E2E tests with CMake 4.2.1 **Note:** Please note that E2E test runs only with `windows-amd64` & `linux-amd64` with cmake version `3.31.5` and `4.2.1` ## Checklist <!-- Put an `x` in the boxes. All tasks must be completed and boxes checked before merging. --> - [ ] 🤖 This change is covered by unit tests (if applicable). - [ ] 🤹 Manual testing has been performed (if necessary). - [x] 🛡️ Security impacts have been considered (if relevant). - [ ] 📖 Documentation updates are complete (if required). - [ ] 🧠 Third-party dependencies and TPIP updated (if required).
1 parent 99dd357 commit 2cc7acf

File tree

2 files changed

+198
-96
lines changed

2 files changed

+198
-96
lines changed

‎.github/workflows/e2e_test.yaml‎

Lines changed: 133 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -21,52 +21,115 @@ jobs:
2121
test: ${{ steps.test-matrix.outputs.matrix }}
2222
steps:
2323
- name: Harden Runner
24-
uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
24+
uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
2525
with:
2626
egress-policy: audit
2727

28-
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
28+
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
2929

3030
- uses: JoshuaTheMiller/conditional-build-matrix@81b51eb8d89e07b86404934b5fecde1cea1163a5 # v2.0.1
31-
id: test-matrix
31+
id: test-matrix-base
3232
with:
3333
inputFile: '.github/matrix_includes.json'
3434
filter: '[?runTests==`true`]'
3535

36+
37+
- name: Generate matrix with cmake versions
38+
id: test-matrix
39+
shell: bash
40+
run: |
41+
base_matrix='${{ steps.test-matrix-base.outputs.matrix }}'
42+
43+
# Read CMake version from vcpkg config
44+
vcpkg_cmake_version=$(jq -r '.requires."arm:tools/kitware/cmake"' ./test/vcpkg-configuration.json)
45+
echo "CMake version from vcpkg config: $vcpkg_cmake_version"
46+
47+
cmake_versions="[\"$vcpkg_cmake_version\", \"4.2.1\"]"
48+
49+
# base_matrix is expected to be: { "include": [ ... ] }
50+
# produce: { "include": [ {..config.., cmake_version: ..}, ... ] }
51+
result=$(jq -nc \
52+
--argjson base "$base_matrix" \
53+
--argjson cmake "$cmake_versions" '
54+
{ include: [ $base.include[] as $cfg | $cmake[] as $ver | ($cfg + {cmake_version: $ver}) ] }
55+
')
56+
57+
echo "matrix=$result" >> $GITHUB_OUTPUT
58+
3659
test:
3760
needs: [ matrix_prep ]
3861
runs-on: ${{ matrix.runs_on }}
3962
continue-on-error: true
4063
strategy:
64+
fail-fast: false
4165
matrix: ${{ fromJson(needs.matrix_prep.outputs.test) }}
66+
name: test(${{ matrix.runs_on }},${{ matrix.arch }},CMake ${{ matrix.cmake_version }})
4267

4368
steps:
4469
- name: Harden Runner
45-
uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
70+
uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
4671
with:
4772
egress-policy: audit
4873

49-
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
74+
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
5075

5176
- name: Set up Python
52-
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
77+
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
5378
with:
5479
python-version: '3.11'
5580
cache: 'pip'
5681

5782
- name: Install pip dependencies
83+
shell: bash
84+
run: |
85+
pip install --upgrade pip
86+
pip install -r test/e2e/requirements.txt
87+
88+
- name: Create vcpkg config without CMake
89+
if: matrix.cmake_version != '3.31.5'
90+
shell: bash
5891
run: |
59-
pip install --upgrade pip
60-
pip install -r test/e2e/requirements.txt
92+
jq 'del(.requires."arm:tools/kitware/cmake")' ./test/vcpkg-configuration.json > ./test/vcpkg-configuration-temp.json
6193
62-
- name: Setup vcpkg environment
94+
- name: Setup vcpkg environment (with CMake)
95+
if: matrix.cmake_version == '3.31.5'
6396
uses: ARM-software/cmsis-actions/vcpkg@afc8e1a46fad8a5e1a08f8477b71050d442f60a7 # v1
6497
with:
6598
config: "./test/vcpkg-configuration.json"
6699
vcpkg-downloads: "${{ github.workspace }}/.vcpkg/downloads"
67100
cache: "-"
68101

102+
- name: Setup vcpkg environment (without CMake)
103+
if: matrix.cmake_version != '3.31.5'
104+
uses: ARM-software/cmsis-actions/vcpkg@afc8e1a46fad8a5e1a08f8477b71050d442f60a7 # v1
105+
with:
106+
config: "./test/vcpkg-configuration-temp.json"
107+
vcpkg-downloads: "${{ github.workspace }}/.vcpkg/downloads"
108+
cache: "-"
109+
110+
# Overlay CMake for non-vcpkg runs (no need to mutate the vcpkg config)
111+
- name: Install CMake ${{ matrix.cmake_version }} (amd64)
112+
if: matrix.cmake_version != '3.31.5' && matrix.arch == 'amd64'
113+
uses: jwlawson/actions-setup-cmake@802fa1a2c4e212495c05bf94dba2704a92a472be # v2.0.2
114+
with:
115+
cmake-version: '${{ matrix.cmake_version }}'
116+
117+
- name: Install CMake ${{ matrix.cmake_version }} (arm64)
118+
if: matrix.cmake_version != '3.31.5' && matrix.arch == 'arm64'
119+
shell: bash
120+
run: |
121+
# Install CMake via pip for ARM64 compatibility
122+
pip install cmake==${{ matrix.cmake_version }}
123+
124+
- name: Confirm CMake selected
125+
shell: bash
126+
run: |
127+
echo "cmake_version matrix value: ${{ matrix.cmake_version }}"
128+
which cmake || true
129+
cmake --version
130+
69131
- name: Activate Arm tool license
132+
shell: bash
70133
run: |
71134
armlm activate --server https://mdk-preview.keil.arm.com --product KEMDK-COM0
72135
working-directory: ./test
@@ -87,72 +150,79 @@ jobs:
87150
env:
88151
IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_TOKEN }}
89152
run: |
90-
robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} \
91-
--variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}.md \
92-
--consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }} \
93-
--name ${{ matrix.target }}-${{ matrix.arch }} \
153+
robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }}-cmake${{ matrix.cmake_version }} \
154+
--variable TEST_ENV_FILE:test-env-${{ matrix.target }}-${{ matrix.arch }}-cmake${{ matrix.cmake_version }}.md \
155+
--consolewidth=150 --settag ${{ matrix.target }}-${{ matrix.arch }}-cmake${{ matrix.cmake_version }} \
156+
--name ${{ matrix.target }}-${{ matrix.arch }}-cmake${{ matrix.cmake_version }} \
94157
./e2e
95158
96-
- name: Archieve test results
159+
- name: Archive test results
97160
if: always()
98-
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
161+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
99162
with:
100-
name: reports-${{ matrix.target }}-${{ matrix.arch }}
101-
path: ./test/reports-${{ matrix.target }}-${{ matrix.arch }}
163+
name: reports-${{ matrix.target }}-${{ matrix.arch }}-cmake${{ matrix.cmake_version }}
164+
path: ./test/reports-${{ matrix.target }}-${{ matrix.arch }}-cmake${{ matrix.cmake_version }}
102165

103166
report:
104167
runs-on: ubuntu-latest
105-
if: always()
106-
needs: test
168+
if: always() && needs.test.result != 'skipped'
169+
needs: [ test ]
107170
permissions: write-all
108171
steps:
109-
- name: Harden Runner
110-
uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
111-
with:
112-
egress-policy: audit
172+
- name: Harden Runner
173+
uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
174+
with:
175+
egress-policy: audit
113176

114-
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
177+
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
115178

116-
- name: Set up Python
117-
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
118-
with:
119-
python-version: '3.11'
120-
cache: 'pip'
179+
- name: Set up Python
180+
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
181+
with:
182+
python-version: '3.11'
183+
cache: 'pip'
121184

122-
- name: Install pip dependencies
123-
run: |
185+
- name: Install pip dependencies
186+
shell: bash
187+
run: |
124188
pip install --upgrade pip
125189
pip install -r test/e2e/requirements.txt
126190
127-
- name: Download reports
128-
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
129-
with:
130-
path: artifacts
131-
pattern: reports-*
132-
133-
- name: Consolidate robot test results
134-
working-directory: artifacts
135-
continue-on-error: true
136-
run: |
137-
python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
138-
./reports-windows-amd64/output.xml \
139-
./reports-linux-amd64/output.xml
140-
141-
- name: Generate Summary report
142-
if: always()
143-
run: |
144-
python ./test/e2e/lib/execution_summary.py artifacts \
145-
-r ./test/e2e/reference.md \
146-
-o artifacts/collective_robot_results/output.xml \
147-
-m summary_report.md
148-
149-
- name: Print E2E Report
150-
if: always()
151-
run: cat summary_report.md >> $GITHUB_STEP_SUMMARY
152-
153-
- name: Archieve consolidated test results
154-
if: always()
155-
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
156-
with:
157-
name: consolidated-reports
158-
path: artifacts/collective_robot_results
191+
- name: Download reports
192+
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
193+
with:
194+
path: artifacts
195+
pattern: reports-*
196+
197+
- name: Consolidate robot test results
198+
working-directory: artifacts
199+
continue-on-error: true
200+
run: |
201+
python -m robot.rebot --name Collective_Robot_Results \
202+
--outputdir collective_robot_results \
203+
--output output.xml \
204+
./reports-windows-amd64-cmake3.31.5/output.xml \
205+
./reports-windows-amd64-cmake4.2.1/output.xml \
206+
./reports-linux-amd64-cmake3.31.5/output.xml \
207+
./reports-linux-amd64-cmake4.2.1/output.xml
208+
209+
- name: Generate Summary report
210+
if: always()
211+
shell: bash
212+
run: |
213+
python ./test/e2e/lib/execution_summary.py artifacts \
214+
-r ./test/e2e/reference.md \
215+
-o artifacts/collective_robot_results/output.xml \
216+
-m summary_report.md
217+
218+
- name: Print E2E Report
219+
if: always()
220+
shell: bash
221+
run: cat summary_report.md >> $GITHUB_STEP_SUMMARY
222+
223+
- name: Archive consolidated test results
224+
if: always()
225+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
226+
with:
227+
name: consolidated-reports
228+
path: artifacts/collective_robot_results

‎test/e2e/reference.md‎

Lines changed: 65 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -4,41 +4,73 @@
44

55
|:white_check_mark: Passed|:x: Failed|:fast_forward: Skipped|Total|
66
|:----:|:----:|:-----:|:---:|
7-
|32|0|0|32|
7+
|64|0|0|64|
88

99
## Passed Tests
1010

1111
|Tag|Test|:clock1030: Duration|Suite|
1212
|:---:|:---:|:---:|:---:|
13-
|windows-amd64|Validate build-asm Example|25.77 s|Local Example Tests|
14-
|windows-amd64|Validate build-c Example|6.94 s|Local Example Tests|
15-
|windows-amd64|Validate build-cpp Example|10.70 s|Local Example Tests|
16-
|windows-amd64|Validate build-set Example|4.23 s|Local Example Tests|
17-
|windows-amd64|Validate executes Example|6.11 s|Local Example Tests|
18-
|windows-amd64|Validate include-define Example|7.85 s|Local Example Tests|
19-
|windows-amd64|Validate language-scope Example|10.17 s|Local Example Tests|
20-
|windows-amd64|Validate linker-pre-processing Example|7.09 s|Local Example Tests|
21-
|windows-amd64|Validate pre-include Example|7.22 s|Local Example Tests|
22-
|windows-amd64|Validate whitespace Example|6.97 s|Local Example Tests|
23-
|windows-amd64|Validate trustzone Example|48.04 s|Local Example Tests|
24-
|windows-amd64|Hello_FRDM-K32L3A6|35.11 s|Remote Example Tests|
25-
|windows-amd64|Hello_LPCXpresso55S69|30.37 s|Remote Example Tests|
26-
|windows-amd64|Csolution-examples|27.52 s|Remote Example Tests|
27-
|windows-amd64|keil-studio-get-started|8.24 s|Remote Example Tests|
28-
|windows-amd64|AVH-VSI|65.48 s|Remote Example Tests|
29-
|linux-amd64|Validate build-asm Example|8.28 s|Local Example Tests|
30-
|linux-amd64|Validate build-c Example|1.67 s|Local Example Tests|
31-
|linux-amd64|Validate build-cpp Example|2.66 s|Local Example Tests|
32-
|linux-amd64|Validate build-set Example|1.12 s|Local Example Tests|
33-
|linux-amd64|Validate executes Example|1.33 s|Local Example Tests|
34-
|linux-amd64|Validate include-define Example|1.93 s|Local Example Tests|
35-
|linux-amd64|Validate language-scope Example|2.48 s|Local Example Tests|
36-
|linux-amd64|Validate linker-pre-processing Example|1.69 s|Local Example Tests|
37-
|linux-amd64|Validate pre-include Example|1.78 s|Local Example Tests|
38-
|linux-amd64|Validate whitespace Example|1.67 s|Local Example Tests|
39-
|linux-amd64|Validate trustzone Example|7.48 s|Local Example Tests|
40-
|linux-amd64|Hello_FRDM-K32L3A6|13.17 s|Remote Example Tests|
41-
|linux-amd64|Hello_LPCXpresso55S69|10.68 s|Remote Example Tests|
42-
|linux-amd64|Csolution-examples|14.93 s|Remote Example Tests|
43-
|linux-amd64|keil-studio-get-started|3.92 s|Remote Example Tests|
44-
|linux-amd64|AVH-VSI|27.26 s|Remote Example Tests|
13+
|windows-amd64-cmake3.31.5|Validate build-asm Example|22.30 s|Local Example Tests|
14+
|windows-amd64-cmake3.31.5|Validate build-c Example|9.72 s|Local Example Tests|
15+
|windows-amd64-cmake3.31.5|Validate build-cpp Example|14.60 s|Local Example Tests|
16+
|windows-amd64-cmake3.31.5|Validate build-set Example|4.60 s|Local Example Tests|
17+
|windows-amd64-cmake3.31.5|Validate executes Example|3.81 s|Local Example Tests|
18+
|windows-amd64-cmake3.31.5|Validate include-define Example|13.07 s|Local Example Tests|
19+
|windows-amd64-cmake3.31.5|Validate language-scope Example|13.64 s|Local Example Tests|
20+
|windows-amd64-cmake3.31.5|Validate linker-pre-processing Example|10.10 s|Local Example Tests|
21+
|windows-amd64-cmake3.31.5|Validate pre-include Example|10.55 s|Local Example Tests|
22+
|windows-amd64-cmake3.31.5|Validate whitespace Example|9.90 s|Local Example Tests|
23+
|windows-amd64-cmake3.31.5|Validate trustzone Example|37.85 s|Local Example Tests|
24+
|windows-amd64-cmake3.31.5|Hello_FRDM-K32L3A6|32.06 s|Remote Example Tests|
25+
|windows-amd64-cmake3.31.5|Hello_LPCXpresso55S69|24.64 s|Remote Example Tests|
26+
|windows-amd64-cmake3.31.5|Csolution-examples|116.44 s|Remote Example Tests|
27+
|windows-amd64-cmake3.31.5|keil-studio-get-started|8.19 s|Remote Example Tests|
28+
|windows-amd64-cmake3.31.5|AVH-VSI|73.83 s|Remote Example Tests|
29+
|windows-amd64-cmake4.2.1|Validate build-asm Example|20.91 s|Local Example Tests|
30+
|windows-amd64-cmake4.2.1|Validate build-c Example|9.08 s|Local Example Tests|
31+
|windows-amd64-cmake4.2.1|Validate build-cpp Example|14.15 s|Local Example Tests|
32+
|windows-amd64-cmake4.2.1|Validate build-set Example|4.31 s|Local Example Tests|
33+
|windows-amd64-cmake4.2.1|Validate executes Example|3.63 s|Local Example Tests|
34+
|windows-amd64-cmake4.2.1|Validate include-define Example|13.03 s|Local Example Tests|
35+
|windows-amd64-cmake4.2.1|Validate language-scope Example|12.59 s|Local Example Tests|
36+
|windows-amd64-cmake4.2.1|Validate linker-pre-processing Example|9.09 s|Local Example Tests|
37+
|windows-amd64-cmake4.2.1|Validate pre-include Example|9.22 s|Local Example Tests|
38+
|windows-amd64-cmake4.2.1|Validate whitespace Example|8.85 s|Local Example Tests|
39+
|windows-amd64-cmake4.2.1|Validate trustzone Example|39.44 s|Local Example Tests|
40+
|windows-amd64-cmake4.2.1|Hello_FRDM-K32L3A6|31.20 s|Remote Example Tests|
41+
|windows-amd64-cmake4.2.1|Hello_LPCXpresso55S69|24.12 s|Remote Example Tests|
42+
|windows-amd64-cmake4.2.1|Csolution-examples|113.61 s|Remote Example Tests|
43+
|windows-amd64-cmake4.2.1|keil-studio-get-started|7.86 s|Remote Example Tests|
44+
|windows-amd64-cmake4.2.1|AVH-VSI|73.77 s|Remote Example Tests|
45+
|linux-amd64-cmake3.31.5|Validate build-asm Example|8.88 s|Local Example Tests|
46+
|linux-amd64-cmake3.31.5|Validate build-c Example|3.60 s|Local Example Tests|
47+
|linux-amd64-cmake3.31.5|Validate build-cpp Example|5.47 s|Local Example Tests|
48+
|linux-amd64-cmake3.31.5|Validate build-set Example|1.96 s|Local Example Tests|
49+
|linux-amd64-cmake3.31.5|Validate executes Example|1.46 s|Local Example Tests|
50+
|linux-amd64-cmake3.31.5|Validate include-define Example|4.70 s|Local Example Tests|
51+
|linux-amd64-cmake3.31.5|Validate language-scope Example|4.69 s|Local Example Tests|
52+
|linux-amd64-cmake3.31.5|Validate linker-pre-processing Example|3.59 s|Local Example Tests|
53+
|linux-amd64-cmake3.31.5|Validate pre-include Example|3.79 s|Local Example Tests|
54+
|linux-amd64-cmake3.31.5|Validate whitespace Example|3.57 s|Local Example Tests|
55+
|linux-amd64-cmake3.31.5|Validate trustzone Example|10.58 s|Local Example Tests|
56+
|linux-amd64-cmake3.31.5|Hello_FRDM-K32L3A6|15.06 s|Remote Example Tests|
57+
|linux-amd64-cmake3.31.5|Hello_LPCXpresso55S69|12.66 s|Remote Example Tests|
58+
|linux-amd64-cmake3.31.5|Csolution-examples|65.13 s|Remote Example Tests|
59+
|linux-amd64-cmake3.31.5|keil-studio-get-started|4.85 s|Remote Example Tests|
60+
|linux-amd64-cmake3.31.5|AVH-VSI|33.10 s|Remote Example Tests|
61+
|linux-amd64-cmake4.2.1|Validate build-asm Example|8.97 s|Local Example Tests|
62+
|linux-amd64-cmake4.2.1|Validate build-c Example|4.31 s|Local Example Tests|
63+
|linux-amd64-cmake4.2.1|Validate build-cpp Example|5.21 s|Local Example Tests|
64+
|linux-amd64-cmake4.2.1|Validate build-set Example|2.15 s|Local Example Tests|
65+
|linux-amd64-cmake4.2.1|Validate executes Example|1.52 s|Local Example Tests|
66+
|linux-amd64-cmake4.2.1|Validate include-define Example|4.32 s|Local Example Tests|
67+
|linux-amd64-cmake4.2.1|Validate language-scope Example|4.89 s|Local Example Tests|
68+
|linux-amd64-cmake4.2.1|Validate linker-pre-processing Example|3.83 s|Local Example Tests|
69+
|linux-amd64-cmake4.2.1|Validate pre-include Example|4.01 s|Local Example Tests|
70+
|linux-amd64-cmake4.2.1|Validate whitespace Example|3.71 s|Local Example Tests|
71+
|linux-amd64-cmake4.2.1|Validate trustzone Example|12.16 s|Local Example Tests|
72+
|linux-amd64-cmake4.2.1|Hello_FRDM-K32L3A6|18.19 s|Remote Example Tests|
73+
|linux-amd64-cmake4.2.1|Hello_LPCXpresso55S69|13.01 s|Remote Example Tests|
74+
|linux-amd64-cmake4.2.1|Csolution-examples|69.72 s|Remote Example Tests|
75+
|linux-amd64-cmake4.2.1|keil-studio-get-started|4.74 s|Remote Example Tests|
76+
|linux-amd64-cmake4.2.1|AVH-VSI|33.58 s|Remote Example Tests|

0 commit comments

Comments
 (0)