Fix elliott functional tests (#1387)

* Initial commit

* add group

* Comment out broken tests

* fixes

Author: thegreyd

Makefile

@@ -19,4 +19,7 @@ unit:
 	./.venv/bin/python -m pytest --verbose --color=yes pyartcd/tests/
 	./.venv/bin/python -m pytest --verbose --color=yes ocp-build-data-validator/tests/
 
+functional-elliott:
+	./.venv/bin/python -m pytest --verbose --color=yes elliott/functional_tests/
+
 test: lint unit

elliott/functional_tests/constants.py

@@ -1,5 +1,5 @@
 import sys
 from elliottlib import constants as elliott_constants
 
-ELLIOTT_CMD = [sys.executable, "-m", "elliottlib.cli", "--quiet"]
+ELLIOTT_CMD = [sys.executable, "-m", "elliottlib.cli"]
 ERRATA_TOOL_URL = elliott_constants.errata_url

elliott/functional_tests/test_change_state.py

@@ -4,14 +4,28 @@
 
 
 class ChangeStateTestCase(unittest.TestCase):
-    def test_change_state(self):
-        out = subprocess.check_output(
-            constants.ELLIOTT_CMD
-            + [
-                "--group=openshift-3.9", "change-state", "--state=QE", "--advisory=47924",
-            ],
-        )
-        self.assertIn("current state is SHIPPED_LIVE", out.decode("utf-8"))
+    def test_change_state_fail(self):
+        cmd = constants.ELLIOTT_CMD + [
+            "--group=openshift-4.2",
+            "change-state",
+            "--state=QE",
+            "--advisory=49645",
+        ]
+        result = subprocess.run(cmd, capture_output=True)
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("Cannot change state from SHIPPED_LIVE to QE", result.stderr.decode())
+
+    def test_change_state_from(self):
+        cmd = constants.ELLIOTT_CMD + [
+            "--group=openshift-4.2",
+            "change-state",
+            "--from=NEW_FILES",
+            "--state=QE",
+            "--advisory=49645",
+        ]
+        result = subprocess.run(cmd, capture_output=True)
+        self.assertEqual(result.returncode, 0, msg=f"stdout: {result.stdout.decode()}, stderr: {result.stderr.decode()}")
+        self.assertIn("Current state is not NEW_FILES: SHIPPED_LIVE", result.stdout.decode())
 
 
 if __name__ == '__main__':

elliott/functional_tests/test_find_bugs_qe.py

@@ -5,14 +5,13 @@
 
 class FindBugsQETestCase(unittest.TestCase):
     def test_find_bugs_qe(self):
-        out = subprocess.check_output(
-            constants.ELLIOTT_CMD
-            + [
-                "--assembly=stream", "--group=openshift-4.6", "find-bugs:qe", '--noop'
-            ]
-        )
-        result = out.decode("utf-8")
-        self.assertRegex(result, "Found \\d+ bugs")
+        cmd = constants.ELLIOTT_CMD + [
+            "--assembly=stream", "--group=openshift-4.6", "find-bugs:qe", '--noop'
+        ]
+        result = subprocess.run(cmd, capture_output=True)
+        self.assertEqual(result.returncode, 0,
+                         msg=f"stdout: {result.stdout.decode()}\nstderr: {result.stderr.decode()}")
+        self.assertRegex(result.stderr.decode(), "Found \\d+ bugs")
 
 
 if __name__ == '__main__':

elliott/functional_tests/test_find_bugs_sweep.py

@@ -5,14 +5,15 @@
 
 class FindBugsSweepTestCase(unittest.TestCase):
     def test_sweep_bugs(self):
-        out = subprocess.check_output(
-            constants.ELLIOTT_CMD
-            + [
-                "--group=openshift-4.11", "--assembly=rc.7", "find-bugs:sweep", "--check-builds",
-                "--into-default-advisories", "--noop"
-            ]
-        )
-        self.assertRegex(out.decode("utf-8"), "Found \\d+ bugs")
+        cmd = constants.ELLIOTT_CMD + [
+            "--group=openshift-4.11",
+            "--assembly=4.11.36",  # This assembly has a pinned bug
+            "find-bugs:sweep"
+        ]
+        result = subprocess.run(cmd, capture_output=True)
+        self.assertEqual(result.returncode, 0,
+                         msg=f"stdout: {result.stdout.decode()}\nstderr: {result.stderr.decode()}")
+        self.assertRegex(result.stdout.decode(), "Found \\d+ bugs")
 
 
 if __name__ == '__main__':

elliott/functional_tests/test_find_builds.py

@@ -4,27 +4,27 @@
 
 # This test may start failing once this version is EOL and we either change the
 # ocp-build-data bugzilla schema or all of the non-shipped builds are garbage-collected.
-version = "4.3"
+version = "4.12"
 
 
 class FindBuildsTestCase(unittest.TestCase):
     def test_find_rpms(self):
-        out = subprocess.check_output(
-            constants.ELLIOTT_CMD
-            + [
-                "--assembly=stream", f"--group=openshift-{version}", "find-builds", "--kind=rpm",
-            ]
-        )
-        self.assertIn("may be attached to an advisory", out.decode("utf-8"))
+        cmd = constants.ELLIOTT_CMD + [
+            "--assembly=stream", f"--group=openshift-{version}", "find-builds", "--kind=rpm",
+        ]
+        result = subprocess.run(cmd, capture_output=True)
+        self.assertEqual(result.returncode, 0,
+                         msg=f"stdout: {result.stdout.decode()}\nstderr: {result.stderr.decode()}")
+        self.assertRegex(result.stderr.decode(), "Found \\d+ builds")
 
     def test_find_images(self):
-        out = subprocess.check_output(
-            constants.ELLIOTT_CMD
-            + [
-                f"--group=openshift-{version}", "-i", "openshift-enterprise-cli", "find-builds", "--kind=image",
-            ]
-        )
-        self.assertIn("may be attached to an advisory", out.decode("utf-8"))
+        cmd = constants.ELLIOTT_CMD + [
+            f"--group=openshift-{version}", "-i", "openshift-enterprise-cli", "find-builds", "--kind=image",
+        ]
+        result = subprocess.run(cmd, capture_output=True)
+        self.assertEqual(result.returncode, 0,
+                         msg=f"stdout: {result.stdout.decode()}\nstderr: {result.stderr.decode()}")
+        self.assertRegex(result.stderr.decode(), "Found \\d+ builds")
 
     def test_change_state(self):
         """To attach a build to an advisory, it will be attempted to set the
@@ -40,7 +40,8 @@ def test_change_state(self):
         ]
         result = subprocess.run(command, capture_output=True)
 
-        self.assertEqual(result.returncode, 1)
+        self.assertEqual(result.returncode, 1,
+                         msg=f"stdout: {result.stdout.decode()}\nstderr: {result.stderr.decode()}")
         self.assertIn('Cannot change state', result.stdout.decode())
 
 

elliott/functional_tests/test_get.py

@@ -4,13 +4,19 @@
 
 
 class GetTestCase(unittest.TestCase):
-    def test_get_errutum(self):
-        out = subprocess.check_output(constants.ELLIOTT_CMD + ["get", "49982"])
-        self.assertIn("49982", out.decode("utf-8"))
+    def test_get_erratum(self):
+        cmd = constants.ELLIOTT_CMD + ["get", "49982"]
+        result = subprocess.run(cmd, capture_output=True)
+        self.assertEqual(result.returncode, 0,
+                         msg=f"stdout: {result.stdout.decode()}\nstderr: {result.stderr.decode()}")
+        self.assertIn("49982", result.stdout.decode())
 
-    def test_get_errutum_with_group(self):
-        out = subprocess.check_output(constants.ELLIOTT_CMD + ["--assembly=stream", "--group=openshift-4.2", "get", "--use-default-advisory", "rpm"])
-        self.assertIn(constants.ERRATA_TOOL_URL, out.decode("utf-8"))
+    def test_get_erratum_with_group(self):
+        cmd = constants.ELLIOTT_CMD + ["--assembly=stream", "--group=openshift-4.2", "get", "--use-default-advisory", "rpm"]
+        result = subprocess.run(cmd, capture_output=True)
+        self.assertEqual(result.returncode, 0,
+                         msg=f"stdout: {result.stdout.decode()}\nstderr: {result.stderr.decode()}")
+        self.assertIn(constants.ERRATA_TOOL_URL, result.stdout.decode())
 
 
 if __name__ == '__main__':