diff --git a/.github/workflows/atex-test.yaml b/.github/workflows/atex-test.yaml index 86bb59cbaa46..f87a9b254a48 100644 --- a/.github/workflows/atex-test.yaml +++ b/.github/workflows/atex-test.yaml @@ -67,6 +67,7 @@ jobs: runs-on: ubuntu-latest needs: check_build strategy: + fail-fast: false matrix: centos_stream_major: [8, 9, 10] container: @@ -105,8 +106,7 @@ jobs: python3 tests/run_tests_testingfarm.py \ --contest-dir contest \ --content-dir content-centos-stream${CS_MAJOR} \ - --plan "/plans/daily" \ - --tests "/hardening/host-os/oscap/stig" \ + --plan "/plans/upstream" \ --compose "CentOS-Stream-${CS_MAJOR}" \ --arch x86_64 \ --os-major-version "${CS_MAJOR}" \ @@ -311,6 +311,7 @@ jobs: check_id: ${{ needs.check_build.outputs.check_id }} sha: ${{ needs.check_build.outputs.pr_sha }} status: completed - conclusion: ${{ job.status }} + # Use test job result to determine conclusion - needs.test.result will be 'failure' if any matrix job failed + conclusion: ${{ needs.test.result }} output: | {"summary":"ATEX tests completed. Job: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. View results: ${{ steps.testing_farm_request.outputs.HTML_LINK }}","title":"ATEX Testing Complete"} diff --git a/.packit.yaml b/.packit.yaml index 2e9ab02c8b3a..9692e98b6610 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -23,136 +23,16 @@ jobs: trigger: commit branch: "gh-readonly-queue/.*" -- &test-static-checks +# when modifying this, modify also tests/tmt-plans/ +- &fedora-tests job: tests trigger: pull_request fmf_path: tests/tmt - identifier: /static-checks - tmt_plan: /plans/contest/static-checks$ - targets: - centos-stream-8: {} - centos-stream-9: {} - centos-stream-10: {} - -# when modifying this, modify also tests/tmt-plans/ - -- <<: *test-static-checks identifier: /rpmbuild-ctest-fedora tmt_plan: /plans/contest/rpmbuild-ctest-fedora$ targets: fedora-all: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/anssi_bp28_high - tmt_plan: /plans/contest/hardening/host-os/ansible/anssi_bp28_high$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/bsi - tmt_plan: /plans/contest/hardening/host-os/ansible/bsi$ - targets: - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ccn_advanced - tmt_plan: /plans/contest/hardening/host-os/ansible/ccn_advanced$ - targets: - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis - tmt_plan: /plans/contest/hardening/host-os/ansible/cis$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis_server_l1 - tmt_plan: /plans/contest/hardening/host-os/ansible/cis_server_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis_workstation_l1 - tmt_plan: /plans/contest/hardening/host-os/ansible/cis_workstation_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis_workstation_l2 - tmt_plan: /plans/contest/hardening/host-os/ansible/cis_workstation_l2$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cui - tmt_plan: /plans/contest/hardening/host-os/ansible/cui$ - targets: - centos-stream-8: {} - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/e8 - tmt_plan: /plans/contest/hardening/host-os/ansible/e8$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/hipaa - tmt_plan: /plans/contest/hardening/host-os/ansible/hipaa$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ism_o - tmt_plan: /plans/contest/hardening/host-os/ansible/ism_o$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ism_o_top_secret - tmt_plan: /plans/contest/hardening/host-os/ansible/ism_o_top_secret$ - targets: - centos-stream-10: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ospp - tmt_plan: /plans/contest/hardening/host-os/ansible/ospp$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/pci-dss - tmt_plan: /plans/contest/hardening/host-os/ansible/pci-dss$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/stig - tmt_plan: /plans/contest/hardening/host-os/ansible/stig$ - -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/anssi_bp28_high - tmt_plan: /plans/contest/hardening/host-os/oscap/anssi_bp28_high$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/bsi - tmt_plan: /plans/contest/hardening/host-os/oscap/bsi$ - targets: - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ccn_advanced - tmt_plan: /plans/contest/hardening/host-os/oscap/ccn_advanced$ - targets: - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis - tmt_plan: /plans/contest/hardening/host-os/oscap/cis$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis_server_l1 - tmt_plan: /plans/contest/hardening/host-os/oscap/cis_server_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis_workstation_l1 - tmt_plan: /plans/contest/hardening/host-os/oscap/cis_workstation_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis_workstation_l2 - tmt_plan: /plans/contest/hardening/host-os/oscap/cis_workstation_l2$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cui - tmt_plan: /plans/contest/hardening/host-os/oscap/cui$ - targets: - centos-stream-8: {} - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/e8 - tmt_plan: /plans/contest/hardening/host-os/oscap/e8$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/hipaa - tmt_plan: /plans/contest/hardening/host-os/oscap/hipaa$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ism_o - tmt_plan: /plans/contest/hardening/host-os/oscap/ism_o$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ism_o_top_secret - tmt_plan: /plans/contest/hardening/host-os/oscap/ism_o_top_secret$ - targets: - centos-stream-10: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ospp - tmt_plan: /plans/contest/hardening/host-os/oscap/ospp$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/pci-dss - tmt_plan: /plans/contest/hardening/host-os/oscap/pci-dss$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/stig - tmt_plan: /plans/contest/hardening/host-os/oscap/stig$ -- <<: *test-static-checks +- <<: *fedora-tests identifier: fedora-cis tmt_plan: /plans/fedora-cis$ - targets: - fedora-all: {} diff --git a/tests/run_tests_testingfarm.py b/tests/run_tests_testingfarm.py index 310b6fcc4dbd..5913993bfece 100644 --- a/tests/run_tests_testingfarm.py +++ b/tests/run_tests_testingfarm.py @@ -3,6 +3,10 @@ import sys import time import gzip +import json +import lzma +import atexit +import signal import logging import argparse import contextlib @@ -34,10 +38,12 @@ def parse_args(): def setup_logging(): """Setup logging configuration with console and file handlers.""" + # Log brief info to console, but be verbose in a separate file-based log (uploaded as artifact) console_log = logging.StreamHandler(sys.stderr) console_log.setLevel(logging.INFO) debug_log_fobj = gzip.open("atex_debug.log.gz", "wt") + atexit.register(debug_log_fobj.close) file_log = logging.StreamHandler(debug_log_fobj) file_log.setLevel(logging.DEBUG) @@ -49,11 +55,22 @@ def setup_logging(): force=True, ) - return debug_log_fobj + +def setup_signal_handlers(): + """Setup signal handlers for graceful abort.""" + def abort_on_signal(signum, _): + logger.error(f"got signal {signum}, aborting") + raise SystemExit(1) + + signal.signal(signal.SIGTERM, abort_on_signal) + signal.signal(signal.SIGHUP, abort_on_signal) def main(): """Main function to run tests on Testing Farm.""" + setup_logging() + setup_signal_handlers() + args = parse_args() # Variables exported to tests @@ -63,10 +80,6 @@ def main(): } with contextlib.ExitStack() as stack: - # Setup logging - debug_log_fobj = setup_logging() - stack.enter_context(contextlib.closing(debug_log_fobj)) - # Load FMF tests from contest directory fmf_tests = FMFTests( args.contest_dir, @@ -133,9 +146,18 @@ def main(): logger.info("Test execution completed!") - # Log final output locations - logger.info(f"Results written to: {output_results}") - logger.info(f"Test files in: {output_files}") + # Log final output locations + logger.info(f"Results written to: {output_results}") + logger.info(f"Test files in: {output_files}") + + # Read back the compressed JSON results and exit with non-0 if anything failed + with lzma.open(output_results, "rt") as results: + for line in results: + fields = json.loads(line) + # [platform, status, test name, subtest name, files, note] + if fields[1] in ("fail", "error", "infra"): + logger.warning("failures found in the results, exiting with 1") + sys.exit(1) if __name__ == "__main__": diff --git a/tests/tmt/plans/contest.fmf b/tests/tmt/plans/contest.fmf index 39056e491125..f2c02fcee583 100644 --- a/tests/tmt/plans/contest.fmf +++ b/tests/tmt/plans/contest.fmf @@ -9,122 +9,6 @@ adjust: report: how: html -# -# Hardening via ansible-playbook remediation -# - -/hardening/host-os/ansible/anssi_bp28_high: - discover+: {test: /hardening/host-os/ansible/anssi_bp28_high$} - -/hardening/host-os/ansible/bsi: - discover+: {test: /hardening/host-os/ansible/bsi$} - -/hardening/host-os/ansible/ccn_advanced: - discover+: {test: /hardening/host-os/ansible/ccn_advanced$} - -/hardening/host-os/ansible/cis: - discover+: {test: /hardening/host-os/ansible/cis$} - -/hardening/host-os/ansible/cis_server_l1: - discover+: {test: /hardening/host-os/ansible/cis_server_l1$} - -/hardening/host-os/ansible/cis_workstation_l1: - discover+: {test: /hardening/host-os/ansible/cis_workstation_l1$} - -/hardening/host-os/ansible/cis_workstation_l2: - discover+: {test: /hardening/host-os/ansible/cis_workstation_l2$} - -/hardening/host-os/ansible/cui: - discover+: {test: /hardening/host-os/ansible/cui$} - -/hardening/host-os/ansible/e8: - discover+: {test: /hardening/host-os/ansible/e8$} - -/hardening/host-os/ansible/hipaa: - discover+: {test: /hardening/host-os/ansible/hipaa$} - -/hardening/host-os/ansible/ism_o: - discover+: {test: /hardening/host-os/ansible/ism_o$} - -/hardening/host-os/ansible/ism_o_top_secret: - discover+: {test: /hardening/host-os/ansible/ism_o_top_secret$} - -/hardening/host-os/ansible/ospp: - discover+: {test: /hardening/host-os/ansible/ospp$} - -/hardening/host-os/ansible/pci-dss: - discover+: {test: /hardening/host-os/ansible/pci-dss$} - -/hardening/host-os/ansible/stig: - discover+: {test: /hardening/host-os/ansible/stig$} - -# -# Hardening via oscap xccdf eval --remediate -# - -/hardening/host-os/oscap/anssi_bp28_high: - discover+: {test: /hardening/host-os/oscap/anssi_bp28_high$} - -/hardening/host-os/oscap/bsi: - discover+: {test: /hardening/host-os/oscap/bsi$} - -/hardening/host-os/oscap/ccn_advanced: - discover+: {test: /hardening/host-os/oscap/ccn_advanced$} - -/hardening/host-os/oscap/cis: - discover+: {test: /hardening/host-os/oscap/cis$} - -/hardening/host-os/oscap/cis_server_l1: - discover+: {test: /hardening/host-os/oscap/cis_server_l1$} - -/hardening/host-os/oscap/cis_workstation_l1: - discover+: {test: /hardening/host-os/oscap/cis_workstation_l1$} - -/hardening/host-os/oscap/cis_workstation_l2: - discover+: {test: /hardening/host-os/oscap/cis_workstation_l2$} - -/hardening/host-os/oscap/cui: - discover+: {test: /hardening/host-os/oscap/cui$} - -/hardening/host-os/oscap/e8: - discover+: {test: /hardening/host-os/oscap/e8$} - -/hardening/host-os/oscap/hipaa: - discover+: {test: /hardening/host-os/oscap/hipaa$} - -/hardening/host-os/oscap/ism_o: - discover+: {test: /hardening/host-os/oscap/ism_o$} - -/hardening/host-os/oscap/ism_o_top_secret: - discover+: {test: /hardening/host-os/oscap/ism_o_top_secret$} - -/hardening/host-os/oscap/ospp: - discover+: {test: /hardening/host-os/oscap/ospp$} - -/hardening/host-os/oscap/pci-dss: - discover+: {test: /hardening/host-os/oscap/pci-dss$} - -/hardening/host-os/oscap/stig: - discover+: {test: /hardening/host-os/oscap/stig$} - -# -# Misc smoke/sanity tests -# - -/static-checks: - discover+: - test: /static-checks - exclude: - # exclude here due to the test failing frequently for short periods - # of time, as many websites have temporary availability issues - - /static-checks/html-links - # these always fail, meant for manual review - - /static-checks/diff - # The value of this test is debatable and therefore it should not delay upstream gating. - # Our SCAP datastream is often noncompliant from the start, for example by containing SCE checks. - - /static-checks/nist-validation - - # Fedora specific plan /rpmbuild-ctest-fedora: discover+: {test: /static-checks/rpmbuild-ctest}