Skip to content

Commit ac08aca

Browse files
nashifnashif
committed
ci: twister: use workflow_call for prep job
Use workflow_call to allow for different type of nodes depending on event. On push, we do not need to use zephyr runners, GH runners are enough and are much faster to deploy and start. This resolves an issue where push jobs will have to be queued for a longer time waiting for the prep step, once the prep step is done, we will have to wait one more time in the queue for requested nodes. This should speed up execution of push events in CI. Signed-off-by: Anas Nashif <[email protected]>
1 parent 0a8b166 commit ac08aca

File tree

2 files changed

+151
-102
lines changed

2 files changed

+151
-102
lines changed

‎.github/workflows/twister-prep.yaml‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
name: Prep
2+
3+
on:
4+
workflow_call:
5+
outputs:
6+
subset:
7+
description: subset
8+
value: ${{ jobs.prep_push.outputs.subset }} || ${{ jobs.prep_pr.outputs.subset }}
9+
size:
10+
description: size
11+
value: ${{ jobs.prep_push.outputs.size }} || ${{ jobs.prep_pr.outputs.size }}
12+
fullrun:
13+
description: fullrun
14+
value: ${{ jobs.prep_push.outputs.fullrun }} || ${{ jobs.prep_pr.outputs.size }}
15+
16+
jobs:
17+
prep_pr:
18+
if: github.repository_owner == 'zephyrproject-rtos' && github.event_name == 'pull_request_target'
19+
runs-on:
20+
group: zephyr-runner-v2-linux-x64-4xlarge
21+
container:
22+
image: ghcr.io/zephyrproject-rtos/ci-repo-cache:v0.27.4.20241026
23+
options: '--entrypoint /bin/bash'
24+
outputs:
25+
subset: ${{ steps.output-services.outputs.subset }}
26+
size: ${{ steps.output-services.outputs.size }}
27+
fullrun: ${{ steps.output-services.outputs.fullrun }}
28+
env:
29+
MATRIX_SIZE: 10
30+
PUSH_MATRIX_SIZE: 20
31+
DAILY_MATRIX_SIZE: 80
32+
BSIM_OUT_PATH: /opt/bsim/
33+
BSIM_COMPONENTS_PATH: /opt/bsim/components
34+
TESTS_PER_BUILDER: 700
35+
COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
36+
BASE_REF: ${{ github.base_ref }}
37+
steps:
38+
- name: Apply container owner mismatch workaround
39+
run: |
40+
# FIXME: The owner UID of the GITHUB_WORKSPACE directory may not
41+
# match the container user UID because of the way GitHub
42+
# Actions runner is implemented. Remove this workaround when
43+
# GitHub comes up with a fundamental fix for this problem.
44+
git config --global --add safe.directory ${GITHUB_WORKSPACE}
45+
46+
- name: Print cloud service information
47+
run: |
48+
echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"
49+
echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"
50+
echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"
51+
52+
- name: Clone cached Zephyr repository
53+
if: github.event_name == 'pull_request_target'
54+
continue-on-error: true
55+
run: |
56+
git clone --shared /repo-cache/zephyrproject/zephyr .
57+
git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}
58+
59+
- name: Checkout
60+
if: github.event_name == 'pull_request_target'
61+
uses: actions/checkout@v4
62+
with:
63+
ref: ${{ github.event.pull_request.head.sha }}
64+
fetch-depth: 0
65+
persist-credentials: false
66+
67+
- name: Environment Setup
68+
if: github.event_name == 'pull_request_target'
69+
run: |
70+
git config --global user.email "[email protected]"
71+
git config --global user.name "Zephyr Bot"
72+
rm -fr ".git/rebase-apply"
73+
git rebase origin/${BASE_REF}
74+
git clean -f -d
75+
git log --pretty=oneline | head -n 10
76+
west init -l . || true
77+
west config manifest.group-filter -- +ci,+optional
78+
west config --global update.narrow true
79+
west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || ( rm -rf ../modules ../bootloader ../tools && west update --path-cache /repo-cache/zephyrproject)
80+
west forall -c 'git reset --hard HEAD'
81+
82+
echo "ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-$( cat SDK_VERSION )" >> $GITHUB_ENV
83+
84+
- name: Generate Test Plan with Twister
85+
if: github.event_name == 'pull_request_target'
86+
id: test-plan
87+
run: |
88+
export ZEPHYR_BASE=${PWD}
89+
export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
90+
python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request -t $TESTS_PER_BUILDER
91+
if [ -s .testplan ]; then
92+
cat .testplan >> $GITHUB_ENV
93+
else
94+
echo "TWISTER_NODES=${MATRIX_SIZE}" >> $GITHUB_ENV
95+
fi
96+
rm -f testplan.json .testplan
97+
98+
- name: Determine matrix size
99+
id: output-services
100+
run: |
101+
if [ -n "${TWISTER_NODES}" ]; then
102+
subset="[$(seq -s',' 1 ${TWISTER_NODES})]"
103+
else
104+
subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"
105+
fi
106+
size=${TWISTER_NODES}
107+
108+
echo "subset=${subset}" >> $GITHUB_OUTPUT
109+
echo "size=${size}" >> $GITHUB_OUTPUT
110+
echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT
111+
112+
prep_push:
113+
if: github.repository_owner == 'zephyrproject-rtos' && (github.event_name == 'push' || github.event_name == 'schedule')
114+
runs-on: ubuntu-22.04
115+
outputs:
116+
subset: ${{ steps.output-services.outputs.subset }}
117+
size: ${{ steps.output-services.outputs.size }}
118+
fullrun: ${{ steps.output-services.outputs.fullrun }}
119+
env:
120+
MATRIX_SIZE: 10
121+
PUSH_MATRIX_SIZE: 20
122+
DAILY_MATRIX_SIZE: 80
123+
BSIM_OUT_PATH: /opt/bsim/
124+
BSIM_COMPONENTS_PATH: /opt/bsim/components
125+
TESTS_PER_BUILDER: 700
126+
COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
127+
BASE_REF: ${{ github.base_ref }}
128+
steps:
129+
- name: Print cloud service information
130+
run: |
131+
echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"
132+
echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"
133+
echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"
134+
135+
- name: Determine matrix size
136+
id: output-services
137+
run: |
138+
if [ "${{github.event_name}}" = "push" ]; then
139+
subset="[$(seq -s',' 1 ${PUSH_MATRIX_SIZE})]"
140+
size=${MATRIX_SIZE}
141+
elif [ "${{github.event_name}}" = "schedule" -a "${{github.repository}}" = "zephyrproject-rtos/zephyr" ]; then
142+
subset="[$(seq -s',' 1 ${DAILY_MATRIX_SIZE})]"
143+
size=${DAILY_MATRIX_SIZE}
144+
else
145+
size=0
146+
fi
147+
148+
echo "subset=${subset}" >> $GITHUB_OUTPUT
149+
echo "size=${size}" >> $GITHUB_OUTPUT
150+
echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT

‎.github/workflows/twister.yaml‎

Lines changed: 1 addition & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -21,108 +21,7 @@ concurrency:
2121

2222
jobs:
2323
twister-build-prep:
24-
if: github.repository_owner == 'zephyrproject-rtos'
25-
runs-on:
26-
group: zephyr-runner-v2-linux-x64-4xlarge
27-
container:
28-
image: ghcr.io/zephyrproject-rtos/ci-repo-cache:v0.27.4.20241026
29-
options: '--entrypoint /bin/bash'
30-
outputs:
31-
subset: ${{ steps.output-services.outputs.subset }}
32-
size: ${{ steps.output-services.outputs.size }}
33-
fullrun: ${{ steps.output-services.outputs.fullrun }}
34-
env:
35-
MATRIX_SIZE: 10
36-
PUSH_MATRIX_SIZE: 20
37-
DAILY_MATRIX_SIZE: 80
38-
BSIM_OUT_PATH: /opt/bsim/
39-
BSIM_COMPONENTS_PATH: /opt/bsim/components
40-
TESTS_PER_BUILDER: 700
41-
COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
42-
BASE_REF: ${{ github.base_ref }}
43-
steps:
44-
- name: Apply container owner mismatch workaround
45-
run: |
46-
# FIXME: The owner UID of the GITHUB_WORKSPACE directory may not
47-
# match the container user UID because of the way GitHub
48-
# Actions runner is implemented. Remove this workaround when
49-
# GitHub comes up with a fundamental fix for this problem.
50-
git config --global --add safe.directory ${GITHUB_WORKSPACE}
51-
52-
- name: Print cloud service information
53-
run: |
54-
echo "ZEPHYR_RUNNER_CLOUD_PROVIDER = ${ZEPHYR_RUNNER_CLOUD_PROVIDER}"
55-
echo "ZEPHYR_RUNNER_CLOUD_NODE = ${ZEPHYR_RUNNER_CLOUD_NODE}"
56-
echo "ZEPHYR_RUNNER_CLOUD_POD = ${ZEPHYR_RUNNER_CLOUD_POD}"
57-
58-
- name: Clone cached Zephyr repository
59-
if: github.event_name == 'pull_request_target'
60-
continue-on-error: true
61-
run: |
62-
git clone --shared /repo-cache/zephyrproject/zephyr .
63-
git remote set-url origin ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}
64-
65-
- name: Checkout
66-
if: github.event_name == 'pull_request_target'
67-
uses: actions/checkout@v4
68-
with:
69-
ref: ${{ github.event.pull_request.head.sha }}
70-
fetch-depth: 0
71-
persist-credentials: false
72-
73-
- name: Environment Setup
74-
if: github.event_name == 'pull_request_target'
75-
run: |
76-
git config --global user.email "[email protected]"
77-
git config --global user.name "Zephyr Bot"
78-
rm -fr ".git/rebase-apply"
79-
git rebase origin/${BASE_REF}
80-
git clean -f -d
81-
git log --pretty=oneline | head -n 10
82-
west init -l . || true
83-
west config manifest.group-filter -- +ci,+optional
84-
west config --global update.narrow true
85-
west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || west update --path-cache /repo-cache/zephyrproject 2>&1 1> west.update.log || ( rm -rf ../modules ../bootloader ../tools && west update --path-cache /repo-cache/zephyrproject)
86-
west forall -c 'git reset --hard HEAD'
87-
88-
echo "ZEPHYR_SDK_INSTALL_DIR=/opt/toolchains/zephyr-sdk-$( cat SDK_VERSION )" >> $GITHUB_ENV
89-
90-
- name: Generate Test Plan with Twister
91-
if: github.event_name == 'pull_request_target'
92-
id: test-plan
93-
run: |
94-
export ZEPHYR_BASE=${PWD}
95-
export ZEPHYR_TOOLCHAIN_VARIANT=zephyr
96-
python3 ./scripts/ci/test_plan.py -c origin/${BASE_REF}.. --pull-request -t $TESTS_PER_BUILDER
97-
if [ -s .testplan ]; then
98-
cat .testplan >> $GITHUB_ENV
99-
else
100-
echo "TWISTER_NODES=${MATRIX_SIZE}" >> $GITHUB_ENV
101-
fi
102-
rm -f testplan.json .testplan
103-
104-
- name: Determine matrix size
105-
id: output-services
106-
run: |
107-
if [ "${{github.event_name}}" = "pull_request_target" ]; then
108-
if [ -n "${TWISTER_NODES}" ]; then
109-
subset="[$(seq -s',' 1 ${TWISTER_NODES})]"
110-
else
111-
subset="[$(seq -s',' 1 ${MATRIX_SIZE})]"
112-
fi
113-
size=${TWISTER_NODES}
114-
elif [ "${{github.event_name}}" = "push" ]; then
115-
subset="[$(seq -s',' 1 ${PUSH_MATRIX_SIZE})]"
116-
size=${MATRIX_SIZE}
117-
elif [ "${{github.event_name}}" = "schedule" -a "${{github.repository}}" = "zephyrproject-rtos/zephyr" ]; then
118-
subset="[$(seq -s',' 1 ${DAILY_MATRIX_SIZE})]"
119-
size=${DAILY_MATRIX_SIZE}
120-
else
121-
size=0
122-
fi
123-
echo "subset=${subset}" >> $GITHUB_OUTPUT
124-
echo "size=${size}" >> $GITHUB_OUTPUT
125-
echo "fullrun=${TWISTER_FULL}" >> $GITHUB_OUTPUT
24+
uses: ./.github/workflows/twister-prep.yaml
12625

12726
twister-build:
12827
runs-on:

0 commit comments

Comments
 (0)