Fully Replace Testing Farm Packit Integration with ATEX integration

.github/workflows/atex-test.yaml

@@ -67,6 +67,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: check_build
     strategy:
+      fail-fast: false
       matrix:
         centos_stream_major: [8, 9, 10]
     container:
@@ -105,8 +106,7 @@ jobs:
           python3 tests/run_tests_testingfarm.py \
             --contest-dir contest \
             --content-dir content-centos-stream${CS_MAJOR} \
-            --plan "/plans/daily" \
-            --tests "/hardening/host-os/oscap/stig" \
+            --plan "/plans/upstream" \
             --compose "CentOS-Stream-${CS_MAJOR}" \
             --arch x86_64 \
             --os-major-version "${CS_MAJOR}" \
@@ -311,6 +311,7 @@ jobs:
           check_id: ${{ needs.check_build.outputs.check_id }}
           sha: ${{ needs.check_build.outputs.pr_sha }}
           status: completed
-          conclusion: ${{ job.status }}
+          # Use test job result to determine conclusion - needs.test.result will be 'failure' if any matrix job failed
+          conclusion: ${{ needs.test.result }}
           output: |
             {"summary":"ATEX tests completed. Job: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. View results: ${{ steps.testing_farm_request.outputs.HTML_LINK }}","title":"ATEX Testing Complete"}

.packit.yaml

@@ -23,136 +23,16 @@ jobs:
   trigger: commit
   branch: "gh-readonly-queue/.*"
 
-- &test-static-checks
+# when modifying this, modify also tests/tmt-plans/
+- &fedora-tests
   job: tests
   trigger: pull_request
   fmf_path: tests/tmt
-  identifier: /static-checks
-  tmt_plan: /plans/contest/static-checks$
-  targets:
-    centos-stream-8: {}
-    centos-stream-9: {}
-    centos-stream-10: {}
-
-# when modifying this, modify also tests/tmt-plans/
-
-- <<: *test-static-checks
   identifier: /rpmbuild-ctest-fedora
   tmt_plan: /plans/contest/rpmbuild-ctest-fedora$
   targets:
     fedora-all: {}
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/anssi_bp28_high
-  tmt_plan: /plans/contest/hardening/host-os/ansible/anssi_bp28_high$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/bsi
-  tmt_plan: /plans/contest/hardening/host-os/ansible/bsi$
-  targets:
-    centos-stream-9: {}
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/ccn_advanced
-  tmt_plan: /plans/contest/hardening/host-os/ansible/ccn_advanced$
-  targets:
-    centos-stream-9: {}
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/cis
-  tmt_plan: /plans/contest/hardening/host-os/ansible/cis$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/cis_server_l1
-  tmt_plan: /plans/contest/hardening/host-os/ansible/cis_server_l1$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/cis_workstation_l1
-  tmt_plan: /plans/contest/hardening/host-os/ansible/cis_workstation_l1$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/cis_workstation_l2
-  tmt_plan: /plans/contest/hardening/host-os/ansible/cis_workstation_l2$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/cui
-  tmt_plan: /plans/contest/hardening/host-os/ansible/cui$
-  targets:
-    centos-stream-8: {}
-    centos-stream-9: {}
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/e8
-  tmt_plan: /plans/contest/hardening/host-os/ansible/e8$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/hipaa
-  tmt_plan: /plans/contest/hardening/host-os/ansible/hipaa$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/ism_o
-  tmt_plan: /plans/contest/hardening/host-os/ansible/ism_o$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/ism_o_top_secret
-  tmt_plan: /plans/contest/hardening/host-os/ansible/ism_o_top_secret$
-  targets:
-    centos-stream-10: {}
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/ospp
-  tmt_plan: /plans/contest/hardening/host-os/ansible/ospp$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/pci-dss
-  tmt_plan: /plans/contest/hardening/host-os/ansible/pci-dss$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/ansible/stig
-  tmt_plan: /plans/contest/hardening/host-os/ansible/stig$
-
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/anssi_bp28_high
-  tmt_plan: /plans/contest/hardening/host-os/oscap/anssi_bp28_high$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/bsi
-  tmt_plan: /plans/contest/hardening/host-os/oscap/bsi$
-  targets:
-    centos-stream-9: {}
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/ccn_advanced
-  tmt_plan: /plans/contest/hardening/host-os/oscap/ccn_advanced$
-  targets:
-    centos-stream-9: {}
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/cis
-  tmt_plan: /plans/contest/hardening/host-os/oscap/cis$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/cis_server_l1
-  tmt_plan: /plans/contest/hardening/host-os/oscap/cis_server_l1$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/cis_workstation_l1
-  tmt_plan: /plans/contest/hardening/host-os/oscap/cis_workstation_l1$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/cis_workstation_l2
-  tmt_plan: /plans/contest/hardening/host-os/oscap/cis_workstation_l2$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/cui
-  tmt_plan: /plans/contest/hardening/host-os/oscap/cui$
-  targets:
-    centos-stream-8: {}
-    centos-stream-9: {}
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/e8
-  tmt_plan: /plans/contest/hardening/host-os/oscap/e8$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/hipaa
-  tmt_plan: /plans/contest/hardening/host-os/oscap/hipaa$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/ism_o
-  tmt_plan: /plans/contest/hardening/host-os/oscap/ism_o$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/ism_o_top_secret
-  tmt_plan: /plans/contest/hardening/host-os/oscap/ism_o_top_secret$
-  targets:
-    centos-stream-10: {}
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/ospp
-  tmt_plan: /plans/contest/hardening/host-os/oscap/ospp$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/pci-dss
-  tmt_plan: /plans/contest/hardening/host-os/oscap/pci-dss$
-- <<: *test-static-checks
-  identifier: /hardening/host-os/oscap/stig
-  tmt_plan: /plans/contest/hardening/host-os/oscap/stig$
 
-- <<: *test-static-checks
+- <<: *fedora-tests
   identifier: fedora-cis
   tmt_plan: /plans/fedora-cis$
-  targets:
-    fedora-all: {}

tests/run_tests_testingfarm.py

@@ -3,6 +3,10 @@
 import sys
 import time
 import gzip
+import json
+import lzma
+import atexit
+import signal
 import logging
 import argparse
 import contextlib
@@ -34,10 +38,12 @@ def parse_args():
 
 def setup_logging():
     """Setup logging configuration with console and file handlers."""
+    # Log brief info to console, but be verbose in a separate file-based log (uploaded as artifact)
     console_log = logging.StreamHandler(sys.stderr)
     console_log.setLevel(logging.INFO)
 
     debug_log_fobj = gzip.open("atex_debug.log.gz", "wt")
+    atexit.register(debug_log_fobj.close)
     file_log = logging.StreamHandler(debug_log_fobj)
     file_log.setLevel(logging.DEBUG)
 
@@ -49,11 +55,22 @@ def setup_logging():
         force=True,
     )
 
-    return debug_log_fobj
+
+def setup_signal_handlers():
+    """Setup signal handlers for graceful abort."""
+    def abort_on_signal(signum, _):
+        logger.error(f"got signal {signum}, aborting")
+        raise SystemExit(1)
+
+    signal.signal(signal.SIGTERM, abort_on_signal)
+    signal.signal(signal.SIGHUP, abort_on_signal)
 
 
 def main():
     """Main function to run tests on Testing Farm."""
+    setup_logging()
+    setup_signal_handlers()
+
     args = parse_args()
 
     # Variables exported to tests
@@ -63,10 +80,6 @@ def main():
     }
 
     with contextlib.ExitStack() as stack:
-        # Setup logging
-        debug_log_fobj = setup_logging()
-        stack.enter_context(contextlib.closing(debug_log_fobj))
-
         # Load FMF tests from contest directory
         fmf_tests = FMFTests(
             args.contest_dir,
@@ -133,9 +146,18 @@ def main():
 
         logger.info("Test execution completed!")
 
-        # Log final output locations
-        logger.info(f"Results written to: {output_results}")
-        logger.info(f"Test files in: {output_files}")
+    # Log final output locations
+    logger.info(f"Results written to: {output_results}")
+    logger.info(f"Test files in: {output_files}")
+
+    # Read back the compressed JSON results and exit with non-0 if anything failed
+    with lzma.open(output_results, "rt") as results:
+        for line in results:
+            fields = json.loads(line)
+            # [platform, status, test name, subtest name, files, note]
+            if fields[1] in ("fail", "error", "infra"):
+                logger.warning("failures found in the results, exiting with 1")
+                sys.exit(1)
 
 
 if __name__ == "__main__":

tests/tmt/plans/contest.fmf

@@ -9,122 +9,6 @@ adjust:
 report:
     how: html
 
-#
-# Hardening via ansible-playbook remediation
-#
-
-/hardening/host-os/ansible/anssi_bp28_high:
-    discover+: {test: /hardening/host-os/ansible/anssi_bp28_high$}
-
-/hardening/host-os/ansible/bsi:
-    discover+: {test: /hardening/host-os/ansible/bsi$}
-
-/hardening/host-os/ansible/ccn_advanced:
-    discover+: {test: /hardening/host-os/ansible/ccn_advanced$}
-
-/hardening/host-os/ansible/cis:
-    discover+: {test: /hardening/host-os/ansible/cis$}
-
-/hardening/host-os/ansible/cis_server_l1:
-    discover+: {test: /hardening/host-os/ansible/cis_server_l1$}
-
-/hardening/host-os/ansible/cis_workstation_l1:
-    discover+: {test: /hardening/host-os/ansible/cis_workstation_l1$}
-
-/hardening/host-os/ansible/cis_workstation_l2:
-    discover+: {test: /hardening/host-os/ansible/cis_workstation_l2$}
-
-/hardening/host-os/ansible/cui:
-    discover+: {test: /hardening/host-os/ansible/cui$}
-
-/hardening/host-os/ansible/e8:
-    discover+: {test: /hardening/host-os/ansible/e8$}
-
-/hardening/host-os/ansible/hipaa:
-    discover+: {test: /hardening/host-os/ansible/hipaa$}
-
-/hardening/host-os/ansible/ism_o:
-    discover+: {test: /hardening/host-os/ansible/ism_o$}
-
-/hardening/host-os/ansible/ism_o_top_secret:
-    discover+: {test: /hardening/host-os/ansible/ism_o_top_secret$}
-
-/hardening/host-os/ansible/ospp:
-    discover+: {test: /hardening/host-os/ansible/ospp$}
-
-/hardening/host-os/ansible/pci-dss:
-    discover+: {test: /hardening/host-os/ansible/pci-dss$}
-
-/hardening/host-os/ansible/stig:
-    discover+: {test: /hardening/host-os/ansible/stig$}
-
-#
-# Hardening via oscap xccdf eval --remediate
-#
-
-/hardening/host-os/oscap/anssi_bp28_high:
-    discover+: {test: /hardening/host-os/oscap/anssi_bp28_high$}
-
-/hardening/host-os/oscap/bsi:
-    discover+: {test: /hardening/host-os/oscap/bsi$}
-
-/hardening/host-os/oscap/ccn_advanced:
-    discover+: {test: /hardening/host-os/oscap/ccn_advanced$}
-
-/hardening/host-os/oscap/cis:
-    discover+: {test: /hardening/host-os/oscap/cis$}
-
-/hardening/host-os/oscap/cis_server_l1:
-    discover+: {test: /hardening/host-os/oscap/cis_server_l1$}
-
-/hardening/host-os/oscap/cis_workstation_l1:
-    discover+: {test: /hardening/host-os/oscap/cis_workstation_l1$}
-
-/hardening/host-os/oscap/cis_workstation_l2:
-    discover+: {test: /hardening/host-os/oscap/cis_workstation_l2$}
-
-/hardening/host-os/oscap/cui:
-    discover+: {test: /hardening/host-os/oscap/cui$}
-
-/hardening/host-os/oscap/e8:
-    discover+: {test: /hardening/host-os/oscap/e8$}
-
-/hardening/host-os/oscap/hipaa:
-    discover+: {test: /hardening/host-os/oscap/hipaa$}
-
-/hardening/host-os/oscap/ism_o:
-    discover+: {test: /hardening/host-os/oscap/ism_o$}
-
-/hardening/host-os/oscap/ism_o_top_secret:
-    discover+: {test: /hardening/host-os/oscap/ism_o_top_secret$}
-
-/hardening/host-os/oscap/ospp:
-    discover+: {test: /hardening/host-os/oscap/ospp$}
-
-/hardening/host-os/oscap/pci-dss:
-    discover+: {test: /hardening/host-os/oscap/pci-dss$}
-
-/hardening/host-os/oscap/stig:
-    discover+: {test: /hardening/host-os/oscap/stig$}
-
-#
-# Misc smoke/sanity tests
-#
-
-/static-checks:
-    discover+:
-        test: /static-checks
-        exclude:
-          # exclude here due to the test failing frequently for short periods
-          # of time, as many websites have temporary availability issues
-          - /static-checks/html-links
-          # these always fail, meant for manual review
-          - /static-checks/diff
-          # The value of this test is debatable and therefore it should not delay upstream gating.
-          # Our SCAP datastream is often noncompliant from the start, for example by containing SCE checks.
-          - /static-checks/nist-validation
-
-
 # Fedora specific plan
 /rpmbuild-ctest-fedora:
     discover+: {test: /static-checks/rpmbuild-ctest}