Merge pull request #862 from moreati/release-0.3.0

Release 0.3.0

Author: moreati

.ci/README.md

@@ -1,8 +1,8 @@
 
 # `.ci`
 
-This directory contains scripts for Travis CI and (more or less) Azure
-Pipelines, but they will also happily run on any Debian-like machine.
+This directory contains scripts for Continuous Integration platforms. Currently
+Azure Pipelines, but they will also happily run on any Debian-like machine.
 
 The scripts are usually split into `_install` and `_test` steps. The `_install`
 step will damage your machine, the `_test` step will just run the tests the way

.ci/ansible_install.py

@@ -6,16 +6,22 @@
     [
         # Must be installed separately, as PyNACL indirect requirement causes
         # newer version to be installed if done in a single pip run.
+        # Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml
         'pip install "pycparser<2.19" "idna<2.7"',
         'pip install '
             '-r tests/requirements.txt '
             '-r tests/ansible/requirements.txt',
+        # encoding is required for installing ansible 2.10 with pip2, otherwise we get a UnicodeDecode error
+        'LC_CTYPE=en_US.UTF-8 LANG=en_US.UTF-8 pip install "ansible-base<2.10.14" "ansible=={}"'.format(ci_lib.ANSIBLE_VERSION)
+    ],
+    [
+        'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
     ]
 ]
 
-batches.extend(
-    ['docker pull %s' % (ci_lib.image_for_distro(distro),)]
+batches[-1].extend([
+    'docker pull %s' % (ci_lib.image_for_distro(distro),)
     for distro in ci_lib.DISTROS
-)
+])
 
 ci_lib.run_batches(batches)

.ci/ansible_tests.py

@@ -37,9 +37,6 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('job_setup'):
-    # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
-    run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION)
-
     os.chdir(TESTS_DIR)
     os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7))
 
@@ -69,13 +66,11 @@ def pause_if_interactive():
         run("sudo apt-get update")
         run("sudo apt-get install -y sshpass")
 
-    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py /usr/lib/python2.7 || true'")
-    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py $VIRTUAL_ENV/lib/python2.7 || true'")
 
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
     try:
-        run('./run_ansible_playbook.py %s -i "%s" %s',
+        run('./run_ansible_playbook.py %s -i "%s" -vvv %s',
             playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
     except:
         pause_if_interactive()

.ci/azure-pipelines-steps.yml

@@ -8,24 +8,17 @@ steps:
 - script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py"
   displayName: "Run prep_azure.py"
 
-# The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
-# broken symlinks, incorrect permissions and missing codecs. So we use the
-# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
-# stuff into. The virtualenv can probably be removed again, but this was a
-# hard-fought battle and for now I am tired of this crap.
 - script: |
-    sudo ln -fs /usr/bin/python$(python.version) /usr/bin/python
-    /usr/bin/python -m pip install -U virtualenv setuptools wheel
-    /usr/bin/python -m virtualenv /tmp/venv -p /usr/bin/python$(python.version)
     echo "##vso[task.prependpath]/tmp/venv/bin"
 
   displayName: activate venv
 
-- script: .ci/spawn_reverse_shell.py
-  displayName: "Spawn reverse shell"
-
 - script: .ci/$(MODE)_install.py
   displayName: "Run $(MODE)_install.py"
+  env:
+    AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
+    AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
+    AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
 
 - script: .ci/$(MODE)_tests.py
   displayName: "Run $(MODE)_tests.py"

.ci/azure-pipelines.yml

@@ -6,23 +6,35 @@
 jobs:
 
 - job: Mac
+  # vanilla Ansible is really slow
+  timeoutInMinutes: 120
   steps:
   - template: azure-pipelines-steps.yml
   pool:
-    vmImage: macOS-10.13
+    vmImage: macOS-10.15
   strategy:
     matrix:
       Mito27_27:
         python.version: '2.7'
         MODE: mitogen
-      Ans280_27:
+      # TODO: test python3, python3 tests are broken
+      Ans210_27:
         python.version: '2.7'
         MODE: localhost_ansible
+        VER: 2.10.0
+
+      # NOTE: this hangs when ran in Ubuntu 18.04
+      Vanilla_210_27:
+        python.version: '2.7'
+        MODE: localhost_ansible
+        VER: 2.10.0
+        STRATEGY: linear
+        ANSIBLE_SKIP_TAGS: resource_intensive
 
 
 - job: Linux
   pool:
-    vmImage: "Ubuntu 16.04"
+    vmImage: "Ubuntu 18.04"
   steps:
   - template: azure-pipelines-steps.yml
   strategy:
@@ -33,7 +45,7 @@ jobs:
       Mito27Debian_27:
         python.version: '2.7'
         MODE: mitogen
-        DISTRO: debian
+        DISTRO: debian9
 
       #MitoPy27CentOS6_26:
         #python.version: '2.7'
@@ -45,9 +57,16 @@ jobs:
         MODE: mitogen
         DISTRO: centos6
 
-      #
-      #
-      #
+      Mito37Debian_27:
+        python.version: '3.7'
+        MODE: mitogen
+        DISTRO: debian9
+
+      Mito39Debian_27:
+        python.version: '3.9'
+        MODE: mitogen
+        DISTRO: debian9
+        VER: 2.10.0
 
       #Py26CentOS7:
         #python.version: '2.7'
@@ -91,12 +110,17 @@ jobs:
         #DISTROS: debian
         #STRATEGY: linear
 
-      Ansible_280_27:
+      Ansible_210_27:
         python.version: '2.7'
         MODE: ansible
-        VER: 2.8.0
+        VER: 2.10.0
 
-      Ansible_280_35:
+      Ansible_210_35:
         python.version: '3.5'
         MODE: ansible
-        VER: 2.8.0
+        VER: 2.10.0
+
+      Ansible_210_39:
+        python.version: '3.9'
+        MODE: ansible
+        VER: 2.10.0