Merge pull request #886 from moreati/prepare-0.3.1

Prepare 0.3.1

Author: moreati

.ci/README.md

@@ -28,8 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `VER`: Ansible version the `_install` script should install. Default changes
-  over time.
 * `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
 * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
   name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`

.ci/ansible_install.py

@@ -3,25 +3,9 @@
 import ci_lib
 
 batches = [
-    [
-        # Must be installed separately, as PyNACL indirect requirement causes
-        # newer version to be installed if done in a single pip run.
-        # Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml
-        'pip install "pycparser<2.19" "idna<2.7"',
-        'pip install '
-            '-r tests/requirements.txt '
-            '-r tests/ansible/requirements.txt',
-        # encoding is required for installing ansible 2.10 with pip2, otherwise we get a UnicodeDecode error
-        'LC_CTYPE=en_US.UTF-8 LANG=en_US.UTF-8 pip install "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION)
-    ],
     [
         'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
     ]
 ]
 
-batches[-1].extend([
-    'docker pull %s' % (ci_lib.image_for_distro(distro),)
-    for distro in ci_lib.DISTROS
-])
-
 ci_lib.run_batches(batches)

.ci/ansible_tests.py

@@ -70,7 +70,7 @@ def pause_if_interactive():
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
     try:
-        run('./run_ansible_playbook.py %s -i "%s" -vvv %s',
+        run('./run_ansible_playbook.py %s -i "%s" %s',
             playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
     except:
         pause_if_interactive()

.ci/azure-pipelines-steps.yml

@@ -5,20 +5,18 @@ parameters:
   sign: false
 
 steps:
-- script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py"
-  displayName: "Run prep_azure.py"
+- task: UsePythonVersion@0
+  displayName: Install python
+  inputs:
+    versionSpec: '$(python.version)'
+  condition: ne(variables['python.version'], '')
 
-- script: |
-    echo "##vso[task.prependpath]/tmp/venv/bin"
+- script: python -mpip install tox
+  displayName: Install tooling
 
-  displayName: activate venv
-
-- script: .ci/$(MODE)_install.py
-  displayName: "Run $(MODE)_install.py"
+- script: python -mtox -e "$(tox.env)"
+  displayName: "Run tests"
   env:
     AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
     AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
     AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
-
-- script: .ci/$(MODE)_tests.py
-  displayName: "Run $(MODE)_tests.py"

.ci/azure-pipelines.yml

@@ -3,75 +3,192 @@
 # Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
 # https://docs.microsoft.com/azure/devops/pipelines/languages/python
 
-jobs:
+# User defined variables are also injected as environment variables
+# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#environment-variables
+#variables:
+  #ANSIBLE_VERBOSITY: 3
 
-- job: Mac
+jobs:
+- job: Mac1015
   # vanilla Ansible is really slow
   timeoutInMinutes: 120
   steps:
   - template: azure-pipelines-steps.yml
   pool:
+    # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md
     vmImage: macOS-10.15
   strategy:
     matrix:
-      Mito27_27:
+      Mito_27:
         python.version: '2.7'
-        MODE: mitogen
+        tox.env: py27-mode_mitogen
+      Mito_36:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen
+      Mito_39:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen
+
       # TODO: test python3, python3 tests are broken
-      Ans210_27:
+      Loc_27_210:
         python.version: '2.7'
-        MODE: localhost_ansible
-        VER: 2.10.0
+        tox.env: py27-mode_localhost-ansible2.10
+      Loc_27_3:
+        python.version: '2.7'
+        tox.env: py27-mode_localhost-ansible3
+      Loc_27_4:
+        python.version: '2.7'
+        tox.env: py27-mode_localhost-ansible4
 
       # NOTE: this hangs when ran in Ubuntu 18.04
-      Vanilla_210_27:
+      Van_27_210:
+        python.version: '2.7'
+        tox.env: py27-mode_localhost-ansible2.10
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
+      Van_27_3:
         python.version: '2.7'
-        MODE: localhost_ansible
-        VER: 2.10.0
+        tox.env: py27-mode_localhost-ansible3
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
+      Van_27_4:
+        python.version: '2.7'
+        tox.env: py27-mode_localhost-ansible4
         STRATEGY: linear
         ANSIBLE_SKIP_TAGS: resource_intensive
 
+- job: Mac11
+  # vanilla Ansible is really slow
+  timeoutInMinutes: 120
+  steps:
+  - template: azure-pipelines-steps.yml
+  pool:
+    # https://github.com/actions/virtual-environments/blob/main/images/macos/
+    vmImage: macOS-11
+  strategy:
+    matrix:
+      Mito_27:
+        tox.env: py27-mode_mitogen
+      Mito_37:
+        python.version: '3.7'
+        tox.env: py37-mode_mitogen
+      Mito_39:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen
+
+      # TODO: test python3, python3 tests are broken
+      Loc_27_210:
+        tox.env: py27-mode_localhost-ansible2.10
+      Loc_27_3:
+        tox.env: py27-mode_localhost-ansible3
+      Loc_27_4:
+        tox.env: py27-mode_localhost-ansible4
+
+      # NOTE: this hangs when ran in Ubuntu 18.04
+      Van_27_210:
+        tox.env: py27-mode_localhost-ansible2.10
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
+      Van_27_3:
+        tox.env: py27-mode_localhost-ansible3
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
+      Van_27_4:
+        tox.env: py27-mode_localhost-ansible4
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
 
 - job: Linux
   pool:
+    # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md
     vmImage: "Ubuntu 18.04"
   steps:
   - template: azure-pipelines-steps.yml
   strategy:
     matrix:
-      #
-      # Confirmed working
-      #
-      Mito27Debian_27:
+      Mito_27_centos6:
         python.version: '2.7'
-        MODE: mitogen
-        DISTRO: debian9
-
-      #MitoPy27CentOS6_26:
-        #python.version: '2.7'
-        #MODE: mitogen
-        #DISTRO: centos6
+        tox.env: py27-mode_mitogen-distro_centos6
+      Mito_27_centos7:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_centos7
+      Mito_27_centos8:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_centos8
+      Mito_27_debian9:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_debian9
+      Mito_27_debian10:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_debian10
+      Mito_27_debian11:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_debian11
+      Mito_27_ubuntu1604:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_ubuntu1604
+      Mito_27_ubuntu1804:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_ubuntu1804
+      Mito_27_ubuntu2004:
+        python.version: '2.7'
+        tox.env: py27-mode_mitogen-distro_ubuntu2004
 
-      Mito36CentOS6_26:
+      Mito_36_centos6:
         python.version: '3.6'
-        MODE: mitogen
-        DISTRO: centos6
-
-      Mito37Debian_27:
-        python.version: '3.7'
-        MODE: mitogen
-        DISTRO: debian9
+        tox.env: py36-mode_mitogen-distro_centos6
+      Mito_36_centos7:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_centos7
+      Mito_36_centos8:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_centos8
+      Mito_36_debian9:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_debian9
+      Mito_36_debian10:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_debian10
+      Mito_36_debian11:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_debian11
+      Mito_36_ubuntu1604:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_ubuntu1604
+      Mito_36_ubuntu1804:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_ubuntu1804
+      Mito_36_ubuntu2004:
+        python.version: '3.6'
+        tox.env: py36-mode_mitogen-distro_ubuntu2004
 
-      Mito39Debian_27:
+      Mito_39_centos6:
         python.version: '3.9'
-        MODE: mitogen
-        DISTRO: debian9
-        VER: 2.10.0
-
-      #Py26CentOS7:
-        #python.version: '2.7'
-        #MODE: mitogen
-        #DISTRO: centos6
+        tox.env: py39-mode_mitogen-distro_centos6
+      Mito_39_centos7:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_centos7
+      Mito_39_centos8:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_centos8
+      Mito_39_debian9:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_debian9
+      Mito_39_debian10:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_debian10
+      Mito_39_debian11:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_debian11
+      Mito_39_ubuntu1604:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_ubuntu1604
+      Mito_39_ubuntu1804:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_ubuntu1804
+      Mito_39_ubuntu2004:
+        python.version: '3.9'
+        tox.env: py39-mode_mitogen-distro_ubuntu2004
 
       #DebOps_2460_27_27:
         #python.version: '2.7'
@@ -110,17 +227,35 @@ jobs:
         #DISTROS: debian
         #STRATEGY: linear
 
-      Ansible_210_27:
+      Ans_27_210:
         python.version: '2.7'
-        MODE: ansible
-        VER: 2.10.0
+        tox.env: py27-mode_ansible-ansible2.10
+      Ans_27_3:
+        python.version: '2.7'
+        tox.env: py27-mode_ansible-ansible3
+      Ans_27_4:
+        python.version: '2.7'
+        tox.env: py27-mode_ansible-ansible4
 
-      Ansible_210_35:
-        python.version: '3.5'
-        MODE: ansible
-        VER: 2.10.0
+      Ans_36_210:
+        python.version: '3.6'
+        tox.env: py36-mode_ansible-ansible2.10
+      Ans_36_3:
+        python.version: '3.6'
+        tox.env: py36-mode_ansible-ansible3
+      Ans_36_4:
+        python.version: '3.6'
+        tox.env: py36-mode_ansible-ansible4
 
-      Ansible_210_39:
+      Ans_39_210:
+        python.version: '3.9'
+        tox.env: py39-mode_ansible-ansible2.10
+      Ans_39_3:
+        python.version: '3.9'
+        tox.env: py39-mode_ansible-ansible3
+      Ans_39_4:
+        python.version: '3.9'
+        tox.env: py39-mode_ansible-ansible4
+      Ans_39_5:
         python.version: '3.9'
-        MODE: ansible
-        VER: 2.10.0
+        tox.env: py39-mode_ansible-ansible5