Support testing of all disabled tests

.github/workflows/disabled-tests.yml

@@ -0,0 +1,211 @@
+# Run all disabled kickstart tests for tested os variants
+name: Run disabled tests
+on:
+  schedule:
+    # run after daily-boot-iso.yml
+    - cron: 0 4 * * *
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  scenario:
+    name: Disabled tests
+    runs-on: [self-hosted, kstest]
+    strategy:
+      matrix:
+        scenario: [daily-iso, rawhide, rhel9, rhel10, centos10]
+      fail-fast: false
+
+    timeout-minutes: 140
+    env:
+      TEST_JOBS: 16
+      GITHUB_TOKEN: /home/github/github-token
+      # The timeout should be ~20 minutes less then the job's timeout-minutes
+      # so that we get partial results and logs in case of the timeout.
+      LAUNCHER_TIMEOUT_MINUTES: 120
+
+    steps:
+      # self-hosted runners don't do this automatically; also useful to keep stuff around for debugging
+      # need to run sudo as the launch script and the container create root/other user owned files
+      - name: Clean up previous run
+        run: |
+          sudo podman ps -q --all --filter='ancestor=kstest-runner' | xargs -tr sudo podman rm -f
+          sudo podman volume rm --all || true
+          sudo rm -rf *
+
+      - name: Clone repository
+        uses: actions/checkout@v4
+        with:
+          path: kickstart-tests
+
+      - name: Generate test cases
+        working-directory: ./kickstart-tests
+        run: scripts/generate-testcases.py -t ./testlib/test_cases/kstest-template.tc.yaml.j2 . -o ./testlib/test_cases
+
+      - name: Clone Permian repository
+        uses: actions/checkout@v4
+        with:
+          repository: rhinstaller/permian
+          path: permian
+          ref: main
+
+      - name: Clone tplib repository
+        uses: actions/checkout@v4
+        with:
+          repository: rhinstaller/tplib
+          path: tplib
+
+      # use the latest official packages for the nightly runs
+      - name: Clean up squid cache
+        run: sudo containers/squid.sh clean
+        working-directory: ./kickstart-tests
+
+      - name: Ensure http proxy is running
+        run: sudo containers/squid.sh start
+        working-directory: ./kickstart-tests
+
+      - name: Generate test selection for os variant ${{ matrix.scenario }}
+        id: generate_query
+        working-directory: ./kickstart-tests
+        run: |
+          LAUNCH_ARGS=$(scripts/generate-launch-args.py --disabled \
+             --os-variant ${{ matrix.scenario }} ) || RC=$?
+          if [ -z ${RC} ] || [ ${RC} == 0 ]; then
+            echo "Generated launch arguments: $LAUNCH_ARGS"
+          else
+            echo "Generating of the arguments failed. See the workflow file for usage."
+            exit 1
+          fi
+          PERMIAN_QUERY=$(scripts/generate-permian-query.py $LAUNCH_ARGS)
+          echo "Generated permian query: $PERMIAN_QUERY"
+          echo "query=$PERMIAN_QUERY" >> $GITHUB_OUTPUT
+          PLATFORM=$(scripts/generate-permian-query.py --print-platform $LAUNCH_ARGS)
+          echo "Generated platform: $PLATFORM"
+          echo "platform=$PLATFORM" >> $GITHUB_OUTPUT
+
+      # Fetch boot.iso and configure its local location
+      - name: Fetch boot.iso if available for os variant ${{ matrix.scenario }}
+        id: boot_iso_for_os_variant
+        run: |
+          set -eux
+          BOOT_ISO_PATH="${{ github.workspace }}/${{ matrix.scenario }}.boot.iso"
+          BOOT_ISO_URL="file://$BOOT_ISO_PATH"
+          if [ "${{ matrix.scenario }}" == "daily-iso" ]; then
+            ${{ github.workspace }}/kickstart-tests/containers/runner/fetch_daily_iso.sh $GITHUB_TOKEN $BOOT_ISO_PATH
+            echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
+          elif [ "${{ matrix.scenario }}" == "rawhide" ]; then
+            curl -L https://download.fedoraproject.org/pub/fedora/linux/development/rawhide/Everything/x86_64/os/images/boot.iso --output $BOOT_ISO_PATH
+            echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
+          else
+            echo "Boot.iso for ${{ matrix.scenario }} can't be fetched."
+            echo "boot_iso=" >> $GITHUB_OUTPUT
+          fi
+
+      # Configure location of installation repositories for the os variant
+      # Also default boot.iso is defined by the value of urls.installation_tree
+      # of kstestParams event structure.
+      - name: Set installation tree for the os variant
+        id: set_installation_urls
+        working-directory: ./kickstart-tests
+        run: |
+          set -eux
+          if [ "${{ matrix.scenario }}" == "rhel8" ] || \
+             [ "${{ matrix.scenario }}" == "rhel9" ] || \
+             [ "${{ matrix.scenario }}" == "rhel10" ] || \
+             [ "${{ matrix.scenario }}" == "centos10" ]; then
+            source ./scripts/defaults-${{ matrix.scenario }}.sh
+            echo "installation_tree=${KSTEST_URL}" >> $GITHUB_OUTPUT
+            echo "modular_url=${KSTEST_MODULAR_URL}" >> $GITHUB_OUTPUT
+          else
+            echo "Installation tree location for ${{ matrix.scenario }} not configured"
+            if [ -z "${{ steps.boot_iso_for_os_variant.outputs.boot_iso }}" ]; then
+              echo "No boot.iso source is defined"
+              exit 2
+            fi
+            echo "installation_tree=" >> $GITHUB_OUTPUT
+            echo "modular_url=" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create Permian settings file
+        working-directory: ./permian
+        run: |
+          cat <<EOF > settings.ini
+          [kickstart_test]
+          kstest_local_repo=${{ github.workspace }}/kickstart-tests
+          [library]
+          directPath=${{ github.workspace }}/kickstart-tests/testlib
+          EOF
+
+      - name: Run kickstart tests in container
+        working-directory: ./permian
+        run: |
+          sudo --preserve-env=TEST_JOBS \
+          PYTHONPATH=${PYTHONPATH:-}:${{ github.workspace }}/tplib \
+          ./run_subset --debug-log permian.log \
+            --settings settings.ini \
+            --override workflows.dry_run=False \
+            --testcase-query '${{ steps.generate_query.outputs.query }}' \
+            run_event '{
+              "type":"everything",
+              "everything_testplan":{
+                "configurations":[{"architecture":"x86_64"}],
+                "point_person":"[email protected]"
+              },
+              ${{ steps.boot_iso_for_os_variant.outputs.boot_iso }}
+              "kstestParams":{
+                "platform":"${{ steps.generate_query.outputs.platform }}",
+                "urls":{
+                  "x86_64":{
+                    "installation_tree":"${{ steps.set_installation_urls.outputs.installation_tree }}",
+                    "modular_url":"${{ steps.set_installation_urls.outputs.modular_url }}"
+                  }
+                }
+              }
+            }'
+
+          # Permian hides the exit code of launcher, so error out this step manually based on logs
+          rc=$( awk '/Runner return code: /{ print $4 }' permian.log)
+          if [ -n "$rc" ]; then
+            exit $rc
+          else
+            grep -q "All execution and reporting is done" permian.log || exit 111
+          fi
+
+      - name: Collect anaconda logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: 'logs-${{ matrix.scenario }}'
+          # skip the /anaconda subdirectories, too large
+          path: |
+            kickstart-tests/data/logs/kstest*.log
+            kickstart-tests/data/logs/kstest.log.json
+            kickstart-tests/data/logs/kstest-*/*.log
+            kickstart-tests/data/logs/kstest-*/anaconda/lorax-packages.log
+            kickstart-tests/data/logs/kstest-*/original-ks.cfg
+
+      - name: Collect json summary
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: 'summary-${{ matrix.scenario }}'
+          path: |
+            kickstart-tests/data/logs/kstest.log.json
+
+      - name: Collect Permian logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: 'logs-permian-${{ matrix.scenario }}'
+          path: |
+            permian/permian.log
+
+      - name: Collect Permian xunit reporter results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: 'results-xunit-${{ matrix.scenario }}'
+          path: |
+            permian/xunit-*.xml

.github/workflows/scenarios-permian.yml

@@ -51,22 +51,12 @@ jobs:
           set -eux
           TESTPLAN="./testlib/test_plans/daily-${{ matrix.scenario }}.plan.yaml"
           TEMPLATE="${TESTPLAN}.j2"
-          if [ "${{ matrix.scenario }}" == "daily-iso" ]; then
-            VARIABLE="SKIP_TESTTYPES_DAILY_ISO"
-          elif [ "${{ matrix.scenario }}" == "rhel8" ]; then
-            VARIABLE="SKIP_TESTTYPES_RHEL8"
-          elif [ "${{ matrix.scenario }}" == "rhel9" ]; then
-            VARIABLE="SKIP_TESTTYPES_RHEL9"
-          elif [ "${{ matrix.scenario }}" == "rhel10" ]; then
-            VARIABLE="SKIP_TESTTYPES_RHEL10"
-          elif [ "${{ matrix.scenario }}" == "centos10" ]; then
-            VARIABLE="SKIP_TESTTYPES_CENTOS10"
-          fi
-          if [ -e ${TEMPLATE} ] && [ -n ${VARIABLE} ]; then
+          OS_VARIANT="${{ matrix.scenario }}"
+          if [ -e ${TEMPLATE} ] && [ -n ${OS_VARIANT} ]; then
             scripts/generate-testplan.py \
               -t ${TEMPLATE} \
               -f ./containers/runner/skip-testtypes \
-              -s ${VARIABLE} \
+              -r ${OS_VARIANT} \
               -o ${TESTPLAN} \
               --verbose
           fi

containers/runner/launch

@@ -50,6 +50,7 @@ Options:
  -j, --jobs N                         Run N jobs in parallel (default: $(recommended_jobs))
  -p, --platform NAME                  See fragments/platform/ (default: fedora_rawhide)
  -t, --testtype TYPE                  Only run TYPE tests
+ -T, --testtypes TYPE[,TYPE..]        Only run tests with any of the given types (comma-delimited)
  -s, --skip-testtypes TYPE[,TYPE..]   Don't run tests with given types
  -u, --updates PATH|URL               Set updates.img path or URL
  -r, --retry                          Retry failed tests once, to guard against random
@@ -74,13 +75,14 @@ EOF
 }
 
 # parse options
-eval set -- "$(getopt -o j:p:t:s:u:rch --long jobs:,platform:,testtype:,skip-testtypes:,updates:,retry,connect-shell,daily-iso:,defaults:,run-args:,recommended-jobs,scenario:,timeout:,dry-run,help -- "$@")"
+eval set -- "$(getopt -o j:p:t:T:s:u:rch --long jobs:,platform:,testtype:,testtypes:,skip-testtypes:,updates:,retry,connect-shell,daily-iso:,defaults:,run-args:,recommended-jobs,scenario:,timeout:,dry-run,help -- "$@")"
 
 while true; do
     case "${1:-}" in
         -j|--jobs) shift; TEST_JOBS=$1 ;;
         -p|--platform) shift; PLATFORM=$1 ;;
         -t|--testtype) shift; TESTTYPE="$1" ;;
+        -T|--testtypes) shift; TESTTYPES="$1" ;;
         -s|--skip-testtypes) shift; SKIP_TESTTYPES="$1" ;;
         -u|--updates) shift; UPDATES_IMAGE="$1" ;;
         -r|--retry) TEST_RETRY=1 ;;
@@ -99,7 +101,7 @@ while true; do
 done
 
 # avoid accidentally running all tests
-if [ -z "${1:-}" ] && [ -z "${TESTTYPE:-}" ]; then
+if [ -z "${1:-}" ] && [ -z "${TESTTYPE:-}" ] && [ -z "${TESTTYPES:-}" ]; then
     usage
     exit 1
 elif [ "${1:-}" = all ] && [ -z "${2:-}" ]; then
@@ -174,7 +176,7 @@ set -x
 $CRUN run -it --rm --device=/dev/kvm --publish 127.0.0.1::16509 $PODMAN_SELINUX_FIX \
     --env SCENARIO="${SCENARIO:-unknown}" \
     --env DRY_RUN="${DRY_RUN:-}" \
-    --env KSTESTS_TEST="$KSTESTS_TEST" --env TESTTYPE="${TESTTYPE:-}" --env SKIP_TESTTYPES="${SKIP_TESTTYPES:-}" \
+    --env KSTESTS_TEST="$KSTESTS_TEST" --env TESTTYPE="${TESTTYPE:-}" --env TESTTYPES="${TESTTYPES:-}" --env SKIP_TESTTYPES="${SKIP_TESTTYPES:-}" \
     --env KSTESTS_RUN_TIMEOUT="${TIMEOUT:-}" \
     --env TEST_JOBS="$TEST_JOBS" --env PLATFORM_NAME="${PLATFORM:-}" --env TEST_RETRY="${TEST_RETRY:-}" ${UPDATES_IMG_ARGS:-} ${CONTAINER_RUN_ARGS:-} \
     --env KSTESTS_KEEP=${KSTESTS_KEEP:-1} \

containers/runner/run-kstest

@@ -41,6 +41,11 @@ if [ -n "${TESTTYPE}" ]; then
     TESTTYPE_ARG="-t ${TESTTYPE}"
 fi
 
+TESTTYPES_ARG=""
+if [ -n "${TESTTYPES}" ]; then
+    TESTTYPES_ARG="-T ${TESTTYPES}"
+fi
+
 SKIP_TESTTYPES_ARG=""
 if [ -n "${SKIP_TESTTYPES}" ]; then
     SKIP_TESTTYPES_ARG="-s ${SKIP_TESTTYPES}"
@@ -81,7 +86,7 @@ fi
 TEST_LOG=/var/tmp/kstest.log
 pushd ${KSTESTS_DIR}
 set +e
-scripts/run_kickstart_tests.sh -x ${TIMEOUT} ${DRY_RUN_ARG} -k ${KSTESTS_KEEP} -i ${ISO_DIR}/${BOOT_ISO} ${UPDATES_IMAGE_ARG} ${PLATFORM_ARG} ${TESTTYPE_ARG} ${SKIP_TESTTYPES_ARG} ${RETRY_ARG} ${KSTESTS_TEST} 2>&1 | tee $TEST_LOG
+scripts/run_kickstart_tests.sh -x ${TIMEOUT} ${DRY_RUN_ARG} -k ${KSTESTS_KEEP} -i ${ISO_DIR}/${BOOT_ISO} ${UPDATES_IMAGE_ARG} ${PLATFORM_ARG} ${TESTTYPE_ARG} ${TESTTYPES_ARG} ${SKIP_TESTTYPES_ARG} ${RETRY_ARG} ${KSTESTS_TEST} 2>&1 | tee $TEST_LOG
 RC=$?
 set -e
 popd