Merge pull request #1159 from moreati/test-distro-specs

moreati · web-flow · commit f384fc33d06b · 2024-10-10T18:54:49.000+01:00
ci: Consolidate Mitogen jobs
diff --git a/.ci/README.md b/.ci/README.md
@@ -28,14 +28,15 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
-  name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
-* `DISTROS`: the `ansible_` tests can run against multiple targets
-  simultaneously, which speeds things up. This is a space-separated list of
-  DISTRO names, but additionally, supports:
+* `MITOGEN_TEST_DISTRO_SPECS`: a space delimited list of distro specs to run
+  the tests against. (e.g. `centos6 ubuntu2004-py3*4`). Each spec determines
+  the Linux distribution, target Python interepreter & number of instances.
+  Only distributions with a pre-built Linux container image can be used.
     * `debian-py3`: when generating Ansible inventory file, set
       `ansible_python_interpreter` to `python3`, i.e. run a test where the
       target interpreter is Python 3.
     * `debian*16`: generate 16 Docker containers running Debian. Also works
       with -py3.
 
+* `MITOGEN_TEST_IMAGE_TEMPLATE`: specifies the Linux container image name,
+  and hence the container registry used for test targets.
diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py
@@ -35,7 +35,7 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.container_specs(ci_lib.DISTROS)
+    containers = ci_lib.container_specs(ci_lib.DISTRO_SPECS.split())
     ci_lib.start_containers(containers)
 
 
diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py
@@ -28,6 +28,10 @@
 )
 
 
+DISTRO_SPECS = os.environ.get(
+    'MITOGEN_TEST_DISTRO_SPECS',
+    'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004',
+)
 IMAGE_TEMPLATE = os.environ.get(
     'MITOGEN_TEST_IMAGE_TEMPLATE',
     'public.ecr.aws/n5z0e8q9/%(distro)s-test',
@@ -196,10 +200,6 @@ def __exit__(self, _1, _2, _3): pass
 
 
 GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-# Used only when MODE=mitogen
-DISTRO = os.environ.get('DISTRO', 'debian9')
-# Used only when MODE=ansible
-DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
 TMP = TempDir().path
 
 
diff --git a/.ci/mitogen_py24_tests.py b/.ci/mitogen_py24_tests.py
@@ -8,8 +8,6 @@
 os.environ.update({
     'NOCOVERAGE': '1',
     'UNIT2': '/usr/local/python2.4.6/bin/unit2',
-
-    'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
     'MITOGEN_LOG_LEVEL': 'debug',
     'SKIP_ANSIBLE': '1',
 })
diff --git a/.ci/mitogen_tests.py b/.ci/mitogen_tests.py
@@ -6,7 +6,6 @@
 import ci_lib
 
 os.environ.update({
-    'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
     'MITOGEN_LOG_LEVEL': 'debug',
     'SKIP_ANSIBLE': '1',
 })
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -67,80 +67,14 @@ jobs:
             python_version: '3.13'
             tox_env: py313-mode_ansible-ansible10-strategy_linear
 
-          - name: Mito_27_centos6
-            tox_env: py27-mode_mitogen-distro_centos6
-          - name: Mito_27_centos7
-            tox_env: py27-mode_mitogen-distro_centos7
-          - name: Mito_27_centos8
-            tox_env: py27-mode_mitogen-distro_centos8
-          - name: Mito_27_debian9
-            tox_env: py27-mode_mitogen-distro_debian9
-          - name: Mito_27_debian10
-            tox_env: py27-mode_mitogen-distro_debian10
-          - name: Mito_27_debian11
-            tox_env: py27-mode_mitogen-distro_debian11
-          - name: Mito_27_ubuntu1604
-            tox_env: py27-mode_mitogen-distro_ubuntu1604
-          - name: Mito_27_ubuntu1804
-            tox_env: py27-mode_mitogen-distro_ubuntu1804
-          - name: Mito_27_ubuntu2004
-            tox_env: py27-mode_mitogen-distro_ubuntu2004
-
-          - name: Mito_36_centos6
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos6
-          - name: Mito_36_centos7
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos7
-          - name: Mito_36_centos8
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_centos8
-          - name: Mito_36_debian9
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian9
-          - name: Mito_36_debian10
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian10
-          - name: Mito_36_debian11
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_debian11
-          - name: Mito_36_ubuntu1604
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu1604
-          - name: Mito_36_ubuntu1804
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu1804
-          - name: Mito_36_ubuntu2004
+          - name: Mito_27
+            tox_env: py27-mode_mitogen
+          - name: Mito_36
             python_version: '3.6'
-            tox_env: py36-mode_mitogen-distro_ubuntu2004
-
-          - name: Mito_313_centos6
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_centos6
-          - name: Mito_313_centos7
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_centos7
-          - name: Mito_313_centos8
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_centos8
-          - name: Mito_313_debian9
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_debian9
-          - name: Mito_313_debian10
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_debian10
-          - name: Mito_313_debian11
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_debian11
-          - name: Mito_313_ubuntu1604
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_ubuntu1604
-          - name: Mito_313_ubuntu1804
-            python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_ubuntu1804
-          - name: Mito_313_ubuntu2004
+            tox_env: py36-mode_mitogen
+          - name: Mito_313
             python_version: '3.13'
-            tox_env: py313-mode_mitogen-distro_ubuntu2004
+            tox_env: py313-mode_mitogen
 
     steps:
       - uses: actions/checkout@v4
diff --git a/docs/changelog.rst b/docs/changelog.rst
@@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file
 In progress (unreleased)
 ------------------------
 
+* :gh:issue:`1159` CI: Reduce number of Jobs by parameterizing Mitogen Docker
+  SSH tests
 
 
 v0.3.13 (2024-10-09)
diff --git a/tests/README.md b/tests/README.md
@@ -30,11 +30,19 @@ and run the tests there.
 1. Run ``test``
 
 
-# Selecting a target distribution
+# Selecting target distributions
 
-Docker target images exist for testing against CentOS and Debian, with the
-default being Debian. To select CentOS, specify `MITOGEN_TEST_DISTRO=centos` in
-the environment.
+Linux container images for testing are available at
+
+- https://github.com/orgs/mitogen-hq/packages
+- https://public.ecr.aws/n5z0e8q9
+
+The images used are determined by two environment variables
+
+- `MITOGEN_TEST_DISTRO_SPECS`
+- `MITOGEN_TEST_IMAGE_TEMPLATE`
+
+Defaults for these can be found in `.ci/ci_lib.py` & `tests/testlib.py`
 
 
 # User Accounts
diff --git a/tests/fakessh_test.py b/tests/fakessh_test.py
@@ -7,8 +7,8 @@
 import testlib
 
 
+@unittest.skip('broken')
 class RsyncTest(testlib.DockerMixin, testlib.TestCase):
-    @unittest.skip('broken')
     def test_rsync_from_master(self):
         context = self.docker_ssh_any()
 
@@ -24,7 +24,6 @@ def test_rsync_from_master(self):
         self.assertTrue(context.call(os.path.exists, '/tmp/data'))
         self.assertTrue(context.call(os.path.exists, '/tmp/data/simple_pkg/a.py'))
 
-    @unittest.skip('broken')
     def test_rsync_between_direct_children(self):
         # master -> SSH -> mitogen__has_sudo_pubkey -> rsync(.ssh) -> master ->
         # mitogen__has_sudo -> rsync
diff --git a/tests/ssh_test.py b/tests/ssh_test.py
@@ -37,7 +37,7 @@ def test_okay(self):
         self.assertEqual(3, context.call(plain_old_module.add, 1, 2))
 
 
-class SshTest(testlib.DockerMixin, testlib.TestCase):
+class SshMixin(testlib.DockerMixin):
     def test_debug_decoding(self):
         # ensure filter_debug_logs() decodes the logged string.
         capture = testlib.LogCapturer()
@@ -68,7 +68,7 @@ def test_stream_name(self):
             password='has_sudo_password',
         )
         name = 'ssh.%s:%s' % (
-            self.dockerized_ssh.get_host(),
+            self.dockerized_ssh.host,
             self.dockerized_ssh.port,
         )
         self.assertEqual(name, context.name)
@@ -176,7 +176,18 @@ def test_accept_enforce_host_keys(self):
             fp.close()
 
 
-class BannerTest(testlib.DockerMixin, testlib.TestCase):
+for distro_spec in testlib.DISTRO_SPECS.split():
+    dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec)
+    klass_name = 'SshTest%s' % (dockerized_ssh.distro.capitalize(),)
+    klass = type(
+        klass_name,
+        (SshMixin, testlib.TestCase),
+        {'dockerized_ssh': dockerized_ssh},
+    )
+    globals()[klass_name] = klass
+
+
+class BannerMixin(testlib.DockerMixin):
     # Verify the ability to disambiguate random spam appearing in the SSHd's
     # login banner from a legitimate password prompt.
     def test_verbose_enabled(self):
@@ -186,13 +197,24 @@ def test_verbose_enabled(self):
             ssh_debug_level=3,
         )
         name = 'ssh.%s:%s' % (
-            self.dockerized_ssh.get_host(),
+            self.dockerized_ssh.host,
             self.dockerized_ssh.port,
         )
         self.assertEqual(name, context.name)
         context.shutdown(wait=True)
 
 
+for distro_spec in testlib.DISTRO_SPECS.split():
+    dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec)
+    klass_name = 'BannerTest%s' % (dockerized_ssh.distro.capitalize(),)
+    klass = type(
+        klass_name,
+        (BannerMixin, testlib.TestCase),
+        {'dockerized_ssh': dockerized_ssh},
+    )
+    globals()[klass_name] = klass
+
+
 class StubPermissionDeniedTest(StubSshMixin, testlib.TestCase):
     def test_classic_prompt(self):
         self.assertRaises(mitogen.ssh.PasswordError,
diff --git a/tests/su_test.py b/tests/su_test.py
@@ -23,7 +23,7 @@ def test_basic(self):
         self.assertEqual(argv[2], '-c')
 
 
-class SuTest(testlib.DockerMixin, testlib.TestCase):
+class SuMixin(testlib.DockerMixin):
     stub_su_path = testlib.data_path('stubs/stub-su.py')
 
     def test_slow_auth_failure(self):
@@ -64,3 +64,14 @@ def test_password_okay(self):
         )
         context = self.router.su(via=ssh, password='rootpassword')
         self.assertEqual(0, context.call(os.getuid))
+
+
+for distro_spec in testlib.DISTRO_SPECS.split():
+    dockerized_ssh = testlib.DockerizedSshDaemon(distro_spec)
+    klass_name = 'SuTest%s' % (dockerized_ssh.distro.capitalize(),)
+    klass = type(
+        klass_name,
+        (SuMixin, testlib.TestCase),
+        {'dockerized_ssh': dockerized_ssh},
+    )
+    globals()[klass_name] = klass
diff --git a/tests/testlib.py b/tests/testlib.py
@@ -51,7 +51,10 @@
 
 LOG = logging.getLogger(__name__)
 
-DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')
+DISTRO_SPECS = os.environ.get(
+    'MITOGEN_TEST_DISTRO_SPECS',
+    'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004',
+)
 IMAGE_TEMPLATE = os.environ.get(
     'MITOGEN_TEST_IMAGE_TEMPLATE',
     'public.ecr.aws/n5z0e8q9/%(distro)s-test',
@@ -555,8 +558,9 @@ def start_container(self):
             self.image,
         ]
         subprocess.check_output(args)
+        self.port = self.get_port(self.container_name)
 
-    def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE):
+    def __init__(self, distro_spec, image_template=IMAGE_TEMPLATE):
         # Code duplicated in ci_lib.py, both should be updated together
         distro_pattern = re.compile(r'''
             (?P<distro>(?P<family>[a-z]+)[0-9]+)
@@ -565,23 +569,21 @@ def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE):
             ''',
             re.VERBOSE,
         )
-        d = distro_pattern.match(distro).groupdict(default=None)
+        d = distro_pattern.match(distro_spec).groupdict(default=None)
+
+        self.distro = d['distro']
+        self.family = d['family']
 
         if d.pop('py') == 'py3':
             self.python_path = '/usr/bin/python3'
         else:
             self.python_path = '/usr/bin/python'
 
         self.image = image_template % d
-        self.start_container()
-        self.host = self.get_host()
-        self.port = self.get_port(self.container_name)
-
-    def get_host(self):
-        return get_docker_host()
+        self.host = get_docker_host()
 
     def wait_for_sshd(self):
-        wait_for_port(self.get_host(), self.port, pattern='OpenSSH')
+        wait_for_port(self.host, self.port, pattern='OpenSSH')
 
     def check_processes(self):
         # Get Accounting name (ucomm) & command line (args) of each process
@@ -651,12 +653,10 @@ def setUpClass(cls):
         if os.environ.get('SKIP_DOCKER_TESTS'):
             raise unittest.SkipTest('SKIP_DOCKER_TESTS is set')
 
-        # we want to be able to override test distro for some tests that need a different container spun up
-        daemon_args = {}
-        if hasattr(cls, 'mitogen_test_distro'):
-            daemon_args['mitogen_test_distro'] = cls.mitogen_test_distro
-
-        cls.dockerized_ssh = DockerizedSshDaemon(**daemon_args)
+        # cls.dockerized_ssh is injected by dynamically generating TestCase
+        # subclasses.
+        # TODO Bite the bullet, switch to e.g. pytest
+        cls.dockerized_ssh.start_container()
         cls.dockerized_ssh.wait_for_sshd()
 
     @classmethod
diff --git a/tox.ini b/tox.ini
