ci: Reduce number of Jobs by parameterizing Mitogen Docker SSH tests

moreati · moreati · commit 28e08ef94c5a · 2024-10-10T18:37:16.000+01:00
This reduces the number of jobs from 48 to 24. The Mitogen part of the test suite has been parameterized on the Linux container targets to be run against. Both the Ansible tests & Mitogen tests now use the same source of truth to control which targets to use: environment variable MITOGEN_TEST_DISTRO_SPECS. This replaces the two mutually exclusive env vars DISTRO and DISTROS. I've also removed vestgial traces of an unused env var MITOGEN_TEST_DISTRO. Parameterization adapted from https://eli.thegreenplace.net/2014/04/02/dynamically-generating-python-test-cases refs #1058, #1059
diff --git a/.ci/README.md b/.ci/README.md
@@ -28,14 +28,15 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
-  name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
-* `DISTROS`: the `ansible_` tests can run against multiple targets
-  simultaneously, which speeds things up. This is a space-separated list of
-  DISTRO names, but additionally, supports:
+* `MITOGEN_TEST_DISTRO_SPECS`: a space delimited list of distro specs to run
+  the tests against. (e.g. `centos6 ubuntu2004-py3*4`). Each spec determines
+  the Linux distribution, target Python interepreter & number of instances.
+  Only distributions with a pre-built Linux container image can be used.
     * `debian-py3`: when generating Ansible inventory file, set
       `ansible_python_interpreter` to `python3`, i.e. run a test where the
       target interpreter is Python 3.
     * `debian*16`: generate 16 Docker containers running Debian. Also works
       with -py3.
 
+* `MITOGEN_TEST_IMAGE_TEMPLATE`: specifies the Linux container image name,
+  and hence the container registry used for test targets.
diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py
@@ -35,7 +35,7 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.container_specs(ci_lib.DISTROS)
+    containers = ci_lib.container_specs(ci_lib.DISTRO_SPECS.split())
     ci_lib.start_containers(containers)
 
 
diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py
@@ -28,6 +28,10 @@
 )
 
 
+DISTRO_SPECS = os.environ.get(
+    'MITOGEN_TEST_DISTRO_SPECS',
+    'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004',
+)
 IMAGE_TEMPLATE = os.environ.get(
     'MITOGEN_TEST_IMAGE_TEMPLATE',
     'public.ecr.aws/n5z0e8q9/%(distro)s-test',
@@ -196,10 +200,6 @@ def __exit__(self, _1, _2, _3): pass
 
 
 GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-# Used only when MODE=mitogen
-DISTRO = os.environ.get('DISTRO', 'debian9')
-# Used only when MODE=ansible
-DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
 TMP = TempDir().path
 
 
diff --git a/.ci/mitogen_py24_tests.py b/.ci/mitogen_py24_tests.py
@@ -8,8 +8,6 @@
 os.environ.update({
     'NOCOVERAGE': '1',
     'UNIT2': '/usr/local/python2.4.6/bin/unit2',
-
-    'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
     'MITOGEN_LOG_LEVEL': 'debug',
     'SKIP_ANSIBLE': '1',
 })
diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py
@@ -6,7 +6,6 @@
 import ci_lib
 
 os.environ.update({
-    'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
     'MITOGEN_LOG_LEVEL': 'debug',
     'SKIP_ANSIBLE': '1',
 })
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -67,80 +67,14 @@ jobs:
             python_version: '3.13'
             tox_env: py313-mode_ansible-ansible10-strategy_linear
 
-          - name: Mito_27_centos6
-            tox_env: py27-mode_mitogen-distro_centos6
-          - name: Mito_27_centos7
-            tox_env: py27-mode_mitogen-distro_centos7
-          - name: Mito_27_centos8
-            tox_env: py27-mode_mitogen-distro_centos8
-          - name: Mito_27_debian9
-            tox_env: py27-mode_mitogen-distro_debian9
-          - name: Mito_27_debian10
-            tox_env: py27-mode_mitogen-distro_debian10
-          - name: Mito_27_debian11
-            tox_env: py27-mode_mitogen-distro_debian11
-          - name: Mito_27_ubuntu1604
-            tox_env: py27-mode_mitogen-distro_ubuntu1604
-          - name: Mito_27_ubuntu1804
-            tox_env: py27-mode_mitogen-distro_ubuntu1804
-          - name: Mito_27_ubuntu2004
-            tox_env: py27-mode_mitogen-distro_ubuntu2004
-
-          - name: Mito_36_centos6
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos6
-          - name: Mito_36_centos7
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos7
-          - name: Mito_36_centos8
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos8
-          - name: Mito_36_debian9
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian9
-          - name: Mito_36_debian10
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian10
-          - name: Mito_36_debian11
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian11
-          - name: Mito_36_ubuntu1604
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu1604
-          - name: Mito_36_ubuntu1804
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu1804
-          - name: Mito_36_ubuntu2004
+          - name: Mito_27
+            tox_env: py27-mode_mitogen
+          - name: Mito_36
             python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu2004
-
-          - name: Mito_313_centos6
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_centos6
-          - name: Mito_313_centos7
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_centos7
-          - name: Mito_313_centos8
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_centos8
-          - name: Mito_313_debian9
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_debian9
-          - name: Mito_313_debian10
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_debian10
-          - name: Mito_313_debian11
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_debian11
-          - name: Mito_313_ubuntu1604
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_ubuntu1604
-          - name: Mito_313_ubuntu1804
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_ubuntu1804
-          - name: Mito_313_ubuntu2004
+            tox_env: py36-mode_mitogen
+          - name: Mito_313
             python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_ubuntu2004
+            tox_env: py313-mode_mitogen
 
     steps:
       - uses: actions/checkout@v4
diff --git a/docs/changelog.rst b/docs/changelog.rst
@@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file
 In progress (unreleased)
 ------------------------
 
+* :gh:issue:`1159` CI: Reduce number of Jobs by parameterizing Mitogen Docker
+  SSH tests
 
 
 v0.3.13 (2024-10-09)
diff --git a/tests/README.md b/tests/README.md
@@ -30,11 +30,19 @@ and run the tests there.
 1. Run ``test``
 
 
-# Selecting a target distribution
+# Selecting target distributions
 
-Docker target images exist for testing against CentOS and Debian, with the
-default being Debian. To select CentOS, specify `MITOGEN_TEST_DISTRO=centos` in
-the environment.
+Linux container images for testing are available at
+
+- https://github.com/orgs/mitogen-hq/packages
+- https://public.ecr.aws/n5z0e8q9
+
+The images used are determined by two environment variables
+
+- `MITOGEN_TEST_DISTRO_SPECS`
+- `MITOGEN_TEST_IMAGE_TEMPLATE`
+
+Defaults for these can be found in `.ci/ci_lib.py` & `tests/testlib.py`
 
 
 # User Accounts
diff --git a/tests/fakessh_test.py b/tests/fakessh_test.py
@@ -7,8 +7,8 @@
 import testlib
 
 
+@unittest.skip('broken')
 class RsyncTest(testlib.DockerMixin, testlib.TestCase):
-    @unittest.skip('broken')
     def test_rsync_from_master(self):
         context = self.docker_ssh_any()
 
@@ -24,7 +24,6 @@ def test_rsync_from_master(self):
         self.assertTrue(context.call(os.path.exists, '/tmp/data'))
         self.assertTrue(context.call(os.path.exists, '/tmp/data/simple_pkg/a.py'))
 
-    @unittest.skip('broken')
     def test_rsync_between_direct_children(self):
         # master -> SSH -> mitogen__has_sudo_pubkey -> rsync(.ssh) -> master ->
         # mitogen__has_sudo -> rsync
diff --git a/tests/ssh_test.py b/tests/ssh_test.py
@@ -37,7 +37,7 @@ def test_okay(self):
         self.assertEqual(3, context.call(plain_old_module.add, 1, 2))
 
 
-class SshTest(testlib.DockerMixin, testlib.TestCase):
+class SshMixin(testlib.DockerMixin):
     def test_debug_decoding(self):
         # ensure filter_debug_logs() decodes the logged string.
         capture = testlib.LogCapturer()
@@ -176,7 +176,18 @@ def test_accept_enforce_host_keys(self):
             fp.close()
 
 
-class BannerTest(testlib.DockerMixin, testlib.TestCase):
+for distro_spec in testlib.DISTRO_SPECS.split():
+    dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec)
+    klass_name = 'SshTest%s' % (dockerized_ssh.distro.capitalize(),)
+    klass = type(
+        klass_name,
+        (SshMixin, testlib.TestCase),
+        {'dockerized_ssh': dockerized_ssh},
+    )
+    globals()[klass_name] = klass
+
+
+class BannerMixin(testlib.DockerMixin):
     # Verify the ability to disambiguate random spam appearing in the SSHd's
     # login banner from a legitimate password prompt.
     def test_verbose_enabled(self):
@@ -193,6 +204,17 @@ def test_verbose_enabled(self):
         context.shutdown(wait=True)
 
 
+for distro_spec in testlib.DISTRO_SPECS.split():
+    dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec)
+    klass_name = 'BannerTest%s' % (dockerized_ssh.distro.capitalize(),)
+    klass = type(
+        klass_name,
+        (BannerMixin, testlib.TestCase),
+        {'dockerized_ssh': dockerized_ssh},
+    )
+    globals()[klass_name] = klass
+
+
 class StubPermissionDeniedTest(StubSshMixin, testlib.TestCase):
     def test_classic_prompt(self):
         self.assertRaises(mitogen.ssh.PasswordError,
diff --git a/tests/su_test.py b/tests/su_test.py
@@ -23,7 +23,7 @@ def test_basic(self):
         self.assertEqual(argv[2], '-c')
 
 
-class SuTest(testlib.DockerMixin, testlib.TestCase):
+class SuMixin(testlib.DockerMixin):
     stub_su_path = testlib.data_path('stubs/stub-su.py')
 
     def test_slow_auth_failure(self):
@@ -64,3 +64,14 @@ def test_password_okay(self):
         )
         context = self.router.su(via=ssh, password='rootpassword')
         self.assertEqual(0, context.call(os.getuid))
+
+
+for distro_spec in testlib.DISTRO_SPECS.split():
+    dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec)
+    klass_name = 'SuTest%s' % (dockerized_ssh.distro.capitalize(),)
+    klass = type(
+        klass_name,
+        (SuMixin, testlib.TestCase),
+        {'dockerized_ssh': dockerized_ssh},
+    )
+    globals()[klass_name] = klass
diff --git a/tests/testlib.py b/tests/testlib.py
@@ -51,7 +51,10 @@
 
 LOG = logging.getLogger(__name__)
 
-DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')
+DISTRO_SPECS = os.environ.get(
+    'MITOGEN_TEST_DISTRO_SPECS',
+    'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004',
+)
 IMAGE_TEMPLATE = os.environ.get(
     'MITOGEN_TEST_IMAGE_TEMPLATE',
     'public.ecr.aws/n5z0e8q9/%(distro)s-test',
@@ -555,8 +558,9 @@ def start_container(self):
             self.image,
         ]
         subprocess.check_output(args)
+        self.port = self.get_port(self.container_name)
 
-    def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE):
+    def __init__(self, distro_spec, image_template=IMAGE_TEMPLATE):
         # Code duplicated in ci_lib.py, both should be updated together
         distro_pattern = re.compile(r'''
             (?P<distro>(?P<family>[a-z]+)[0-9]+)
@@ -565,17 +569,18 @@ def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE):
             ''',
             re.VERBOSE,
         )
-        d = distro_pattern.match(distro).groupdict(default=None)
+        d = distro_pattern.match(distro_spec).groupdict(default=None)
+
+        self.distro = d['distro']
+        self.family = d['family']
 
         if d.pop('py') == 'py3':
             self.python_path = '/usr/bin/python3'
         else:
             self.python_path = '/usr/bin/python'
 
         self.image = image_template % d
-        self.start_container()
         self.host = get_docker_host()
-        self.port = self.get_port(self.container_name)
 
     def wait_for_sshd(self):
         wait_for_port(self.host, self.port, pattern='OpenSSH')
@@ -648,12 +653,10 @@ def setUpClass(cls):
         if os.environ.get('SKIP_DOCKER_TESTS'):
             raise unittest.SkipTest('SKIP_DOCKER_TESTS is set')
 
-        # we want to be able to override test distro for some tests that need a different container spun up
-        daemon_args = {}
-        if hasattr(cls, 'mitogen_test_distro'):
-            daemon_args['mitogen_test_distro'] = cls.mitogen_test_distro
-
-        cls.dockerized_ssh = DockerizedSshDaemon(**daemon_args)
+        # cls.dockerized_ssh is injected by dynamically generating TestCase
+        # subclasses.
+        # TODO Bite the bullet, switch to e.g. pytest
+        cls.dockerized_ssh.start_container()
         cls.dockerized_ssh.wait_for_sshd()
 
     @classmethod
diff --git a/tox.ini b/tox.ini
