Merge pull request openshift#6966 from shiftstack/openstack_manifest_junit

openstack-manifests: Export JUnit results

Author: openshift-merge-robot

hack/openstack/test-manifests.sh

@@ -89,6 +89,14 @@ while getopts a:c:e:f:i:t:pr:h o; do
 done
 readonly api_fip os_cloud external_network compute_flavor openshift_install tests_dir persist run
 
+declare python_venv
+if [[ -w './bin' ]]; then
+	python_venv="$(realpath './bin/venv')"
+else
+	python_venv="$(mktemp -d)"
+fi
+readonly python_venv
+
 declare -a temp_dirs
 cleanup() {
 	if [[ "$persist" == 'NO' ]]; then
@@ -101,13 +109,21 @@ trap cleanup EXIT
 
 validate_configuration
 
+python -m venv "$python_venv"
+# shellcheck source=/dev/null
+source "${python_venv}/bin/activate"
+pip install unittest-xml-reporting pyyaml
+
 >&2 echo "Running the tests from '${tests_dir}' against the Installer binary '${openshift_install}'."
 
+export JUNIT_DIR="${ARTIFACT_DIR:-.}/junit"
+mkdir -p "$JUNIT_DIR"
+
 declare result='PASS'
 for testcase in "${tests_dir}"/* ; do
 	if [ -d "$testcase" ] && [[ "$testcase" =~ $run ]]; then
 		echo
-		echo "*** TEST CASE: $(basename "${testcase}")"
+		echo "*** TEST CASE: $(basename -- "$testcase")"
 		assets_dir="$(mktemp -d)"
 		if [[ "$persist" != 'NO' ]]; then
 			echo "Generated assets for this test can be found in ${assets_dir}"
@@ -117,6 +133,12 @@ for testcase in "${tests_dir}"/* ; do
 		fill_install_config "${testcase}/install-config.yaml" > "${assets_dir}/install-config.yaml"
 		"$openshift_install" --log-level warn create manifests --dir "$assets_dir"
 		for t in "${testcase}"/test_*; do
+			declare JUNIT_FILE test_name
+			test_name="$(basename -- "$t")"
+			test_name="${test_name#test_}"
+			test_name="${test_name%.*}"
+			JUNIT_FILE="${JUNIT_DIR}/$(basename -- "${testcase}")_${test_name}.xml"
+			export JUNIT_FILE
 			if $t "$assets_dir"; then
 				echo "PASS: '$t'"
 			else

scripts/openstack/manifest-tests/cinder-availability-zones/test_machines.py

@@ -2,19 +2,25 @@
 # -*- coding: utf-8 -*-
 
 import unittest
+import xmlrunner
 
+import os
 import sys
 import glob
 import yaml
 
 ASSETS_DIR = ""
 
-EXPECTED_MACHINES_NUMBER = 10
+EXPECTED_MASTER_REPLICAS = 10
+EXPECTED_WORKER_REPLICAS = 1000
+
 EXPECTED_MASTER_ZONE_NAMES = ["MasterAZ1", "MasterAZ2", "MasterAZ3"]
+EXPECTED_WORKER_ZONE_NAMES = ["ComputeAZ1", "ComputeAZ2", "ComputeAZ3"]
+
 EXPECTED_VOLUME_ZONE_NAMES = ["VolumeAZ1", "VolumeAZ2", "VolumeAZ3"]
 
 
-class TestVolumeAZMachines(unittest.TestCase):
+class CinderAvailabilityZonesMachines(unittest.TestCase):
     def setUp(self):
         """Parse the Machines into a Python data structure."""
         self.machines = []
@@ -35,7 +41,7 @@ def test_zone_names(self):
 
     def test_total_instance_number(self):
         """Assert that there are as many Machines as required ControlPlane replicas."""
-        self.assertEqual(len(self.machines), EXPECTED_MACHINES_NUMBER)
+        self.assertEqual(len(self.machines), EXPECTED_MASTER_REPLICAS)
 
     def test_replica_distribution(self):
         """Assert that machines are evenly distributed across volume azs."""
@@ -52,6 +58,47 @@ def test_replica_distribution(self):
                 self.assertTrue(-2 < replicas - setpoint < 2)
 
 
+class CinderAvailabilityZonesMachinesets(unittest.TestCase):
+    def setUp(self):
+        """Parse the MachineSets into a Python data structure."""
+        self.machinesets = []
+        for machineset_path in glob.glob(
+                f'{ASSETS_DIR}/openshift/99_openshift-cluster-api_worker-machineset-*.yaml'
+        ):
+            with open(machineset_path) as f:
+                self.machinesets.append(yaml.load(f, Loader=yaml.FullLoader))
+
+    def test_machineset_zone_name(self):
+        """Assert that there is exactly one MachineSet per volume availability zone."""
+        found = []
+        for machineset in self.machinesets:
+            master_zone = machineset["spec"]["template"]["spec"]["providerSpec"]["value"]["availabilityZone"]
+            volume_zone = machineset["spec"]["template"]["spec"]["providerSpec"]["value"]["rootVolume"]["availabilityZone"]
+            self.assertIn(volume_zone, EXPECTED_VOLUME_ZONE_NAMES)
+            self.assertNotIn(volume_zone, found)
+            self.assertEqual(master_zone[-3:], volume_zone[-3:])
+            found.append(volume_zone)
+        self.assertEqual(len(self.machinesets), len(EXPECTED_VOLUME_ZONE_NAMES))
+
+    def test_total_replica_number(self):
+        """Assert that replicas spread across the MachineSets add up to the expected number."""
+        total_found = 0
+        for machineset in self.machinesets:
+            total_found += machineset["spec"]["replicas"]
+        self.assertEqual(total_found, EXPECTED_WORKER_REPLICAS)
+
+    def test_replica_distribution(self):
+        """Assert that replicas are evenly distributed across machinesets."""
+        setpoint = 0
+        for machineset in self.machinesets:
+            replicas = machineset["spec"]["replicas"]
+            if setpoint == 0:
+                setpoint = replicas
+            else:
+                self.assertTrue(-2 < replicas - setpoint < 2)
+
+
 if __name__ == '__main__':
     ASSETS_DIR = sys.argv.pop()
-    unittest.main(verbosity=2)
+    with open(os.environ.get('JUNIT_FILE', '/dev/null'), 'wb') as output:
+        unittest.main(testRunner=xmlrunner.XMLTestRunner(output=output), failfast=False, buffer=False, catchbreak=False, verbosity=2)

scripts/openstack/manifest-tests/cinder-availability-zones/test_machinesets.py

@@ -2,17 +2,18 @@
 # -*- coding: utf-8 -*-
 
 import unittest
+import xmlrunner
 
 import os
 import sys
 import glob
 import yaml
 
 ASSETS_DIR = ""
-INSTALLCONFIG_PATH = ""
+INSTALLCONFIG_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'install-config.yaml')
 
 
-class TestMachines(unittest.TestCase):
+class FailureDomainsMachines(unittest.TestCase):
     def setUp(self):
         """Parse the expected values from install-config and collect Machine resources."""
         self.machines = []
@@ -123,5 +124,5 @@ def test_storage_zone_names(self):
 
 if __name__ == '__main__':
     ASSETS_DIR = sys.argv.pop()
-    INSTALLCONFIG_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'install-config.yaml')
-    unittest.main(verbosity=2)
+    with open(os.environ.get('JUNIT_FILE', '/dev/null'), 'wb') as output:
+        unittest.main(testRunner=xmlrunner.XMLTestRunner(output=output), failfast=False, buffer=False, catchbreak=False, verbosity=2)

scripts/openstack/manifest-tests/failure-domains/test_machines.py

@@ -2,14 +2,16 @@
 # -*- coding: utf-8 -*-
 
 import unittest
+import xmlrunner
 
+import os
 import sys
 import glob
 import yaml
 
 ASSETS_DIR = ""
 
-class TestClusterInfraObject(unittest.TestCase):
+class DefaultStableLoadBalancerClusterInfraObject(unittest.TestCase):
     def setUp(self):
         """Parse the Cluster Infrastructure object into a Python data structure."""
         self.machines = []
@@ -24,4 +26,5 @@ def test_load_balancer(self):
 
 if __name__ == '__main__':
     ASSETS_DIR = sys.argv.pop()
-    unittest.main(verbosity=2)
+    with open(os.environ.get('JUNIT_FILE', '/dev/null'), 'wb') as output:
+        unittest.main(testRunner=xmlrunner.XMLTestRunner(output=output), failfast=False, buffer=False, catchbreak=False, verbosity=2)

…ts/default-stable-lb/install-config.yaml …ts/lb-default-stable/install-config.yamlscripts/openstack/manifest-tests/default-stable-lb/install-config.yaml renamed to scripts/openstack/manifest-tests/lb-default-stable/install-config.yaml scripts/openstack/manifest-tests/default-stable-lb/install-config.yaml renamed to scripts/openstack/manifest-tests/lb-default-stable/install-config.yaml

@@ -2,14 +2,16 @@
 # -*- coding: utf-8 -*-
 
 import unittest
+import xmlrunner
 
+import os
 import sys
 import glob
 import yaml
 
 ASSETS_DIR = ""
 
-class TestClusterInfraObject(unittest.TestCase):
+class DefaultTechPreviewLoadBalancerClusterInfraObject(unittest.TestCase):
     def setUp(self):
         """Parse the Cluster Infrastructure object into a Python data structure."""
         self.machines = []
@@ -24,4 +26,5 @@ def test_load_balancer(self):
 
 if __name__ == '__main__':
     ASSETS_DIR = sys.argv.pop()
-    unittest.main(verbosity=2)
+    with open(os.environ.get('JUNIT_FILE', '/dev/null'), 'wb') as output:
+        unittest.main(testRunner=xmlrunner.XMLTestRunner(output=output), failfast=False, buffer=False, catchbreak=False, verbosity=2)

…/default-stable-lb/test_cluster-infra.py …/lb-default-stable/test_cluster-infra.pyscripts/openstack/manifest-tests/default-stable-lb/test_cluster-infra.py renamed to scripts/openstack/manifest-tests/lb-default-stable/test_cluster-infra.py scripts/openstack/manifest-tests/default-stable-lb/test_cluster-infra.py renamed to scripts/openstack/manifest-tests/lb-default-stable/test_cluster-infra.py

@@ -2,22 +2,24 @@
 # -*- coding: utf-8 -*-
 
 import unittest
+import xmlrunner
 
+import os
 import sys
 import glob
 import yaml
 
 ASSETS_DIR = ""
 
-class TestClusterInfraObject(unittest.TestCase):
+class ManagedLoadBalancer(unittest.TestCase):
     def setUp(self):
         """Parse the Cluster Infrastructure object into a Python data structure."""
         self.machines = []
         cluster_infra = f'{ASSETS_DIR}/manifests/cluster-infrastructure-02-config.yml'
         with open(cluster_infra) as f:
             self.cluster_infra = yaml.load(f, Loader=yaml.FullLoader)
 
-    def test_load_balancer(self):
+    def test_cluster_infra_object(self):
         """Assert that the Cluster infrastructure object contains the LoadBalancer configuration."""
         self.assertIn("loadBalancer", self.cluster_infra["status"]["platformStatus"]["openstack"])
 
@@ -29,4 +31,5 @@ def test_load_balancer(self):
 
 if __name__ == '__main__':
     ASSETS_DIR = sys.argv.pop()
-    unittest.main(verbosity=2)
+    with open(os.environ.get('JUNIT_FILE', '/dev/null'), 'wb') as output:
+        unittest.main(testRunner=xmlrunner.XMLTestRunner(output=output), failfast=False, buffer=False, catchbreak=False, verbosity=2)