test: fix: stop doing PR A/B-tests across host commands

The output of the spectre/meltdown checker, and the vulnerability files
on the host will not be influenced by the local checkout of the
firecracker repository. Thus these A/B-tests were noops. Fix this by
only doing the non-PR assertion in the nightly pipeline.

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

tests/integration_tests/security/test_vulnerabilities.py

@@ -11,10 +11,10 @@
 import pytest
 import requests
 
+from framework import utils
 from framework.ab_test import (
     git_ab_test_guest_command,
     git_ab_test_guest_command_if_pr,
-    git_ab_test_host_command_if_pr,
     is_pr,
     set_did_not_grow_comparator,
 )
@@ -212,20 +212,17 @@ def check_vulnerabilities_on_guest(status):
     assert report_guest_vulnerabilities == known_guest_vulnerabilities
 
 
+# Nothing can be sensibly tested in a PR context here
+@pytest.mark.skipif(
+    is_pr(), reason="Test depends solely on factors external to GitHub repository"
+)
 def test_spectre_meltdown_checker_on_host(spectre_meltdown_checker):
     """
     Test with the spectre / meltdown checker on host.
     """
-    output = git_ab_test_host_command_if_pr(
-        f"sh {spectre_meltdown_checker} --batch json",
-        comparator=set_did_not_grow_comparator(
-            spectre_meltdown_reported_vulnerablities
-        ),
-        check_in_nonpr=False,
-    )
+    rc, output, _ = utils.run_cmd(f"sh {spectre_meltdown_checker} --batch json")
 
-    # Outside the PR context, checks the return code with some exceptions.
-    if output and output.returncode != 0:
+    if output and rc != 0:
         report = spectre_meltdown_reported_vulnerablities(output)
         expected = {}
         assert report == expected, f"Unexpected vulnerabilities: {report} vs {expected}"
@@ -383,17 +380,15 @@ def get_vuln_files_exception_dict(template):
     return exception_dict
 
 
+# Nothing can be sensibly tested here in a PR context
+@pytest.mark.skipif(
+    is_pr(), reason="Test depends solely on factors external to GitHub repository"
+)
 def test_vulnerabilities_on_host():
     """
     Test vulnerabilities files on host.
     """
-
-    git_ab_test_host_command_if_pr(
-        f"! grep -r Vulnerable {VULN_DIR}",
-        comparator=set_did_not_grow_comparator(
-            lambda output: set(output.stdout.splitlines())
-        ),
-    )
+    utils.check_output(f"! grep -r Vulnerable {VULN_DIR}")
 
 
 def check_vulnerabilities_files_on_guest(microvm):