Merge pull request #1128 from moreati/github-actions

CI: Begin migration from Azure DevOps to GitHub actions

Author: moreati

.ci/README.md

@@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
 * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
   name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
 * `DISTROS`: the `ansible_` tests can run against multiple targets

.ci/ansible_install.py

@@ -35,7 +35,7 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.make_containers()
+    containers = ci_lib.container_specs(ci_lib.DISTROS)
     ci_lib.start_containers(containers)
 
 

.ci/ansible_tests.py

@@ -14,6 +14,19 @@ steps:
     versionSpec: '$(python.version)'
   condition: ne(variables['python.version'], '')
 
+- script: |
+    set -o errexit
+    set -o nounset
+    set -o pipefail
+
+    aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws
+  displayName: Authenticate to container registry
+  condition: eq(variables['Agent.OS'], 'Linux')
+  env:
+    AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
+    AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
+    AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
+
 - script: |
     set -o errexit
     set -o nounset
@@ -90,7 +103,3 @@ steps:
 
     "$PYTHON" -m tox -e "$(tox.env)"
   displayName: "Run tests"
-  env:
-    AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
-    AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
-    AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)

.ci/azure-pipelines-steps.yml

@@ -27,6 +27,13 @@
     )
 )
 
+
+IMAGE_TEMPLATE = os.environ.get(
+    'MITOGEN_TEST_IMAGE_TEMPLATE',
+    'public.ecr.aws/n5z0e8q9/%(distro)s-test',
+)
+
+
 _print = print
 def print(*args, **kwargs):
     file = kwargs.get('file', sys.stdout)
@@ -193,8 +200,6 @@ def __exit__(self, _1, _2, _3): pass
 DISTRO = os.environ.get('DISTRO', 'debian9')
 # Used only when MODE=ansible
 DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
-TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
-BASE_PORT = 2200
 TMP = TempDir().path
 
 
@@ -217,6 +222,7 @@ def __exit__(self, _1, _2, _3): pass
 def get_docker_hostname():
     """Return the hostname where the docker daemon is running.
     """
+    # Duplicated in testlib
     url = os.environ.get('DOCKER_HOST')
     if url in (None, 'http+docker://localunixsocket'):
         return 'localhost'
@@ -225,27 +231,34 @@ def get_docker_hostname():
     return parsed.netloc.partition(':')[0]
 
 
-def make_containers(name_prefix='', port_offset=0):
+def container_specs(
+        distros,
+        base_port=2200,
+        image_template=IMAGE_TEMPLATE,
+        name_template='target-%(distro)s-%(index)d',
+):
     """
     >>> import pprint
-    >>> BASE_PORT=2200; DISTROS=['debian11', 'centos6']
-    >>> pprint.pprint(make_containers())
+    >>> pprint.pprint(container_specs(['debian11-py3', 'centos6']))
     [{'distro': 'debian11',
       'family': 'debian',
       'hostname': 'localhost',
       'image': 'public.ecr.aws/n5z0e8q9/debian11-test',
+      'index': 1,
       'name': 'target-debian11-1',
       'port': 2201,
-      'python_path': '/usr/bin/python'},
+      'python_path': '/usr/bin/python3'},
      {'distro': 'centos6',
       'family': 'centos',
       'hostname': 'localhost',
       'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
+      'index': 2,
       'name': 'target-centos6-2',
       'port': 2202,
       'python_path': '/usr/bin/python'}]
     """
     docker_hostname = get_docker_hostname()
+    # Code duplicated in testlib.py, both should be updated together
     distro_pattern = re.compile(r'''
         (?P<distro>(?P<family>[a-z]+)[0-9]+)
         (?:-(?P<py>py3))?
@@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0):
     i = 1
     lst = []
 
-    for distro in DISTROS:
+    for distro in distros:
+        # Code duplicated in testlib.py, both should be updated together
         d = distro_pattern.match(distro).groupdict(default=None)
-        distro = d['distro']
-        family = d['family']
-        image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
 
-        if d['py'] == 'py3':
+        if d.pop('py') == 'py3':
             python_path = '/usr/bin/python3'
         else:
             python_path = '/usr/bin/python'
 
-        if d['count']:
-            count = int(count)
-        else:
-            count = 1
+        count = int(d.pop('count') or '1', 10)
 
         for x in range(count):
-            lst.append({
-                "distro": distro, "family": family, "image": image,
-                "name": name_prefix + ("target-%s-%s" % (distro, i)),
+            d['index'] = i
+            d.update({
+                'image': image_template % d,
+                'name': name_template % d,
                 "hostname": docker_hostname,
-                "port": BASE_PORT + i + port_offset,
+                'port': base_port + i,
                 "python_path": python_path,
             })
+            lst.append(d)
             i += 1
 
     return lst

.ci/ci_lib.py

@@ -2,16 +2,10 @@
 
 import ci_lib
 
-# Naturally DebOps only supports Debian.
-ci_lib.DISTROS = ['debian']
-
 ci_lib.run_batches([
     [
         'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"',
     ],
-    [
-        'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi',
-    ],
 ])
 
 ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2')

.ci/debops_common_install.py

@@ -6,17 +6,18 @@
 import ci_lib
 
 
-# DebOps only supports Debian.
-ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
-
 project_dir = os.path.join(ci_lib.TMP, 'project')
 vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
 inventory_path = 'ansible/inventory/hosts'
 docker_hostname = ci_lib.get_docker_hostname()
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-')
+    containers = ci_lib.container_specs(
+        ['debian*2'],
+        base_port=2700,
+        name_template='debops-target-%(distro)s-%(index)d',
+    )
     ci_lib.start_containers(containers)
 
 

.ci/debops_common_tests.py

@@ -3,9 +3,6 @@
 import ci_lib
 
 batches = [
-    [
-        'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi',
-    ],
     [
         'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',
     ]