Skip to content

Commit b4d4b6c

Browse files
rvykydalrvykydal
authored
Merge pull request #1512 from rvykydal/add-testtypes-option
Support testing of all disabled tests
2 parents 65a9f3c + 05e54df commit b4d4b6c

File tree

9 files changed

+362
-60
lines changed

9 files changed

+362
-60
lines changed

‎.github/workflows/disabled-tests.yml‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
# Run all disabled kickstart tests for tested os variants
2+
name: Run disabled tests
3+
on:
4+
schedule:
5+
# run after daily-boot-iso.yml
6+
- cron: 0 4 * * *
7+
workflow_dispatch:
8+
9+
permissions:
10+
contents: read
11+
12+
jobs:
13+
scenario:
14+
name: Disabled tests
15+
runs-on: [self-hosted, kstest]
16+
strategy:
17+
matrix:
18+
scenario: [daily-iso, rawhide, rhel9, rhel10, centos10]
19+
fail-fast: false
20+
21+
timeout-minutes: 140
22+
env:
23+
TEST_JOBS: 16
24+
GITHUB_TOKEN: /home/github/github-token
25+
# The timeout should be ~20 minutes less then the job's timeout-minutes
26+
# so that we get partial results and logs in case of the timeout.
27+
LAUNCHER_TIMEOUT_MINUTES: 120
28+
29+
steps:
30+
# self-hosted runners don't do this automatically; also useful to keep stuff around for debugging
31+
# need to run sudo as the launch script and the container create root/other user owned files
32+
- name: Clean up previous run
33+
run: |
34+
sudo podman ps -q --all --filter='ancestor=kstest-runner' | xargs -tr sudo podman rm -f
35+
sudo podman volume rm --all || true
36+
sudo rm -rf *
37+
38+
- name: Clone repository
39+
uses: actions/checkout@v4
40+
with:
41+
path: kickstart-tests
42+
43+
- name: Generate test cases
44+
working-directory: ./kickstart-tests
45+
run: scripts/generate-testcases.py -t ./testlib/test_cases/kstest-template.tc.yaml.j2 . -o ./testlib/test_cases
46+
47+
- name: Clone Permian repository
48+
uses: actions/checkout@v4
49+
with:
50+
repository: rhinstaller/permian
51+
path: permian
52+
ref: main
53+
54+
- name: Clone tplib repository
55+
uses: actions/checkout@v4
56+
with:
57+
repository: rhinstaller/tplib
58+
path: tplib
59+
60+
# use the latest official packages for the nightly runs
61+
- name: Clean up squid cache
62+
run: sudo containers/squid.sh clean
63+
working-directory: ./kickstart-tests
64+
65+
- name: Ensure http proxy is running
66+
run: sudo containers/squid.sh start
67+
working-directory: ./kickstart-tests
68+
69+
- name: Generate test selection for os variant ${{ matrix.scenario }}
70+
id: generate_query
71+
working-directory: ./kickstart-tests
72+
run: |
73+
LAUNCH_ARGS=$(scripts/generate-launch-args.py --disabled \
74+
--os-variant ${{ matrix.scenario }} ) || RC=$?
75+
if [ -z ${RC} ] || [ ${RC} == 0 ]; then
76+
echo "Generated launch arguments: $LAUNCH_ARGS"
77+
else
78+
echo "Generating of the arguments failed. See the workflow file for usage."
79+
exit 1
80+
fi
81+
PERMIAN_QUERY=$(scripts/generate-permian-query.py $LAUNCH_ARGS)
82+
echo "Generated permian query: $PERMIAN_QUERY"
83+
echo "query=$PERMIAN_QUERY" >> $GITHUB_OUTPUT
84+
PLATFORM=$(scripts/generate-permian-query.py --print-platform $LAUNCH_ARGS)
85+
echo "Generated platform: $PLATFORM"
86+
echo "platform=$PLATFORM" >> $GITHUB_OUTPUT
87+
88+
# Fetch boot.iso and configure its local location
89+
- name: Fetch boot.iso if available for os variant ${{ matrix.scenario }}
90+
id: boot_iso_for_os_variant
91+
run: |
92+
set -eux
93+
BOOT_ISO_PATH="${{ github.workspace }}/${{ matrix.scenario }}.boot.iso"
94+
BOOT_ISO_URL="file://$BOOT_ISO_PATH"
95+
if [ "${{ matrix.scenario }}" == "daily-iso" ]; then
96+
${{ github.workspace }}/kickstart-tests/containers/runner/fetch_daily_iso.sh $GITHUB_TOKEN $BOOT_ISO_PATH
97+
echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
98+
elif [ "${{ matrix.scenario }}" == "rawhide" ]; then
99+
curl -L https://download.fedoraproject.org/pub/fedora/linux/development/rawhide/Everything/x86_64/os/images/boot.iso --output $BOOT_ISO_PATH
100+
echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
101+
else
102+
echo "Boot.iso for ${{ matrix.scenario }} can't be fetched."
103+
echo "boot_iso=" >> $GITHUB_OUTPUT
104+
fi
105+
106+
# Configure location of installation repositories for the os variant
107+
# Also default boot.iso is defined by the value of urls.installation_tree
108+
# of kstestParams event structure.
109+
- name: Set installation tree for the os variant
110+
id: set_installation_urls
111+
working-directory: ./kickstart-tests
112+
run: |
113+
set -eux
114+
if [ "${{ matrix.scenario }}" == "rhel8" ] || \
115+
[ "${{ matrix.scenario }}" == "rhel9" ] || \
116+
[ "${{ matrix.scenario }}" == "rhel10" ] || \
117+
[ "${{ matrix.scenario }}" == "centos10" ]; then
118+
source ./scripts/defaults-${{ matrix.scenario }}.sh
119+
echo "installation_tree=${KSTEST_URL}" >> $GITHUB_OUTPUT
120+
echo "modular_url=${KSTEST_MODULAR_URL}" >> $GITHUB_OUTPUT
121+
else
122+
echo "Installation tree location for ${{ matrix.scenario }} not configured"
123+
if [ -z "${{ steps.boot_iso_for_os_variant.outputs.boot_iso }}" ]; then
124+
echo "No boot.iso source is defined"
125+
exit 2
126+
fi
127+
echo "installation_tree=" >> $GITHUB_OUTPUT
128+
echo "modular_url=" >> $GITHUB_OUTPUT
129+
fi
130+
131+
- name: Create Permian settings file
132+
working-directory: ./permian
133+
run: |
134+
cat <<EOF > settings.ini
135+
[kickstart_test]
136+
kstest_local_repo=${{ github.workspace }}/kickstart-tests
137+
[library]
138+
directPath=${{ github.workspace }}/kickstart-tests/testlib
139+
EOF
140+
141+
- name: Run kickstart tests in container
142+
working-directory: ./permian
143+
run: |
144+
sudo --preserve-env=TEST_JOBS \
145+
PYTHONPATH=${PYTHONPATH:-}:${{ github.workspace }}/tplib \
146+
./run_subset --debug-log permian.log \
147+
--settings settings.ini \
148+
--override workflows.dry_run=False \
149+
--testcase-query '${{ steps.generate_query.outputs.query }}' \
150+
run_event '{
151+
"type":"everything",
152+
"everything_testplan":{
153+
"configurations":[{"architecture":"x86_64"}],
154+
"point_person":"[email protected]"
155+
},
156+
${{ steps.boot_iso_for_os_variant.outputs.boot_iso }}
157+
"kstestParams":{
158+
"platform":"${{ steps.generate_query.outputs.platform }}",
159+
"urls":{
160+
"x86_64":{
161+
"installation_tree":"${{ steps.set_installation_urls.outputs.installation_tree }}",
162+
"modular_url":"${{ steps.set_installation_urls.outputs.modular_url }}"
163+
}
164+
}
165+
}
166+
}'
167+
168+
# Permian hides the exit code of launcher, so error out this step manually based on logs
169+
rc=$( awk '/Runner return code: /{ print $4 }' permian.log)
170+
if [ -n "$rc" ]; then
171+
exit $rc
172+
else
173+
grep -q "All execution and reporting is done" permian.log || exit 111
174+
fi
175+
176+
- name: Collect anaconda logs
177+
if: always()
178+
uses: actions/upload-artifact@v4
179+
with:
180+
name: 'logs-${{ matrix.scenario }}'
181+
# skip the /anaconda subdirectories, too large
182+
path: |
183+
kickstart-tests/data/logs/kstest*.log
184+
kickstart-tests/data/logs/kstest.log.json
185+
kickstart-tests/data/logs/kstest-*/*.log
186+
kickstart-tests/data/logs/kstest-*/anaconda/lorax-packages.log
187+
kickstart-tests/data/logs/kstest-*/original-ks.cfg
188+
189+
- name: Collect json summary
190+
if: always()
191+
uses: actions/upload-artifact@v4
192+
with:
193+
name: 'summary-${{ matrix.scenario }}'
194+
path: |
195+
kickstart-tests/data/logs/kstest.log.json
196+
197+
- name: Collect Permian logs
198+
if: always()
199+
uses: actions/upload-artifact@v4
200+
with:
201+
name: 'logs-permian-${{ matrix.scenario }}'
202+
path: |
203+
permian/permian.log
204+
205+
- name: Collect Permian xunit reporter results
206+
if: always()
207+
uses: actions/upload-artifact@v4
208+
with:
209+
name: 'results-xunit-${{ matrix.scenario }}'
210+
path: |
211+
permian/xunit-*.xml

‎.github/workflows/scenarios-permian.yml‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -51,22 +51,12 @@ jobs:
5151
set -eux
5252
TESTPLAN="./testlib/test_plans/daily-${{ matrix.scenario }}.plan.yaml"
5353
TEMPLATE="${TESTPLAN}.j2"
54-
if [ "${{ matrix.scenario }}" == "daily-iso" ]; then
55-
VARIABLE="SKIP_TESTTYPES_DAILY_ISO"
56-
elif [ "${{ matrix.scenario }}" == "rhel8" ]; then
57-
VARIABLE="SKIP_TESTTYPES_RHEL8"
58-
elif [ "${{ matrix.scenario }}" == "rhel9" ]; then
59-
VARIABLE="SKIP_TESTTYPES_RHEL9"
60-
elif [ "${{ matrix.scenario }}" == "rhel10" ]; then
61-
VARIABLE="SKIP_TESTTYPES_RHEL10"
62-
elif [ "${{ matrix.scenario }}" == "centos10" ]; then
63-
VARIABLE="SKIP_TESTTYPES_CENTOS10"
64-
fi
65-
if [ -e ${TEMPLATE} ] && [ -n ${VARIABLE} ]; then
54+
OS_VARIANT="${{ matrix.scenario }}"
55+
if [ -e ${TEMPLATE} ] && [ -n ${OS_VARIANT} ]; then
6656
scripts/generate-testplan.py \
6757
-t ${TEMPLATE} \
6858
-f ./containers/runner/skip-testtypes \
69-
-s ${VARIABLE} \
59+
-r ${OS_VARIANT} \
7060
-o ${TESTPLAN} \
7161
--verbose
7262
fi

‎containers/runner/launch‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ Options:
5050
-j, --jobs N Run N jobs in parallel (default: $(recommended_jobs))
5151
-p, --platform NAME See fragments/platform/ (default: fedora_rawhide)
5252
-t, --testtype TYPE Only run TYPE tests
53+
-T, --testtypes TYPE[,TYPE..] Only run tests with any of the given types (comma-delimited)
5354
-s, --skip-testtypes TYPE[,TYPE..] Don't run tests with given types
5455
-u, --updates PATH|URL Set updates.img path or URL
5556
-r, --retry Retry failed tests once, to guard against random
@@ -74,13 +75,14 @@ EOF
7475
}
7576

7677
# parse options
77-
eval set -- "$(getopt -o j:p:t:s:u:rch --long jobs:,platform:,testtype:,skip-testtypes:,updates:,retry,connect-shell,daily-iso:,defaults:,run-args:,recommended-jobs,scenario:,timeout:,dry-run,help -- "$@")"
78+
eval set -- "$(getopt -o j:p:t:T:s:u:rch --long jobs:,platform:,testtype:,testtypes:,skip-testtypes:,updates:,retry,connect-shell,daily-iso:,defaults:,run-args:,recommended-jobs,scenario:,timeout:,dry-run,help -- "$@")"
7879

7980
while true; do
8081
case "${1:-}" in
8182
-j|--jobs) shift; TEST_JOBS=$1 ;;
8283
-p|--platform) shift; PLATFORM=$1 ;;
8384
-t|--testtype) shift; TESTTYPE="$1" ;;
85+
-T|--testtypes) shift; TESTTYPES="$1" ;;
8486
-s|--skip-testtypes) shift; SKIP_TESTTYPES="$1" ;;
8587
-u|--updates) shift; UPDATES_IMAGE="$1" ;;
8688
-r|--retry) TEST_RETRY=1 ;;
@@ -99,7 +101,7 @@ while true; do
99101
done
100102

101103
# avoid accidentally running all tests
102-
if [ -z "${1:-}" ] && [ -z "${TESTTYPE:-}" ]; then
104+
if [ -z "${1:-}" ] && [ -z "${TESTTYPE:-}" ] && [ -z "${TESTTYPES:-}" ]; then
103105
usage
104106
exit 1
105107
elif [ "${1:-}" = all ] && [ -z "${2:-}" ]; then
@@ -174,7 +176,7 @@ set -x
174176
$CRUN run -it --rm --device=/dev/kvm --publish 127.0.0.1::16509 $PODMAN_SELINUX_FIX \
175177
--env SCENARIO="${SCENARIO:-unknown}" \
176178
--env DRY_RUN="${DRY_RUN:-}" \
177-
--env KSTESTS_TEST="$KSTESTS_TEST" --env TESTTYPE="${TESTTYPE:-}" --env SKIP_TESTTYPES="${SKIP_TESTTYPES:-}" \
179+
--env KSTESTS_TEST="$KSTESTS_TEST" --env TESTTYPE="${TESTTYPE:-}" --env TESTTYPES="${TESTTYPES:-}" --env SKIP_TESTTYPES="${SKIP_TESTTYPES:-}" \
178180
--env KSTESTS_RUN_TIMEOUT="${TIMEOUT:-}" \
179181
--env TEST_JOBS="$TEST_JOBS" --env PLATFORM_NAME="${PLATFORM:-}" --env TEST_RETRY="${TEST_RETRY:-}" ${UPDATES_IMG_ARGS:-} ${CONTAINER_RUN_ARGS:-} \
180182
--env KSTESTS_KEEP=${KSTESTS_KEEP:-1} \

‎containers/runner/run-kstest‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ if [ -n "${TESTTYPE}" ]; then
4141
TESTTYPE_ARG="-t ${TESTTYPE}"
4242
fi
4343

44+
TESTTYPES_ARG=""
45+
if [ -n "${TESTTYPES}" ]; then
46+
TESTTYPES_ARG="-T ${TESTTYPES}"
47+
fi
48+
4449
SKIP_TESTTYPES_ARG=""
4550
if [ -n "${SKIP_TESTTYPES}" ]; then
4651
SKIP_TESTTYPES_ARG="-s ${SKIP_TESTTYPES}"
@@ -81,7 +86,7 @@ fi
8186
TEST_LOG=/var/tmp/kstest.log
8287
pushd ${KSTESTS_DIR}
8388
set +e
84-
scripts/run_kickstart_tests.sh -x ${TIMEOUT} ${DRY_RUN_ARG} -k ${KSTESTS_KEEP} -i ${ISO_DIR}/${BOOT_ISO} ${UPDATES_IMAGE_ARG} ${PLATFORM_ARG} ${TESTTYPE_ARG} ${SKIP_TESTTYPES_ARG} ${RETRY_ARG} ${KSTESTS_TEST} 2>&1 | tee $TEST_LOG
89+
scripts/run_kickstart_tests.sh -x ${TIMEOUT} ${DRY_RUN_ARG} -k ${KSTESTS_KEEP} -i ${ISO_DIR}/${BOOT_ISO} ${UPDATES_IMAGE_ARG} ${PLATFORM_ARG} ${TESTTYPE_ARG} ${TESTTYPES_ARG} ${SKIP_TESTTYPES_ARG} ${RETRY_ARG} ${KSTESTS_TEST} 2>&1 | tee $TEST_LOG
8590
RC=$?
8691
set -e
8792
popd

0 commit comments

Comments
 (0)