Merge pull request #816 from moreati/issue815

Expand coverage of mitogen/<distro>-test docker images

Author: s1113950

.ci/azure-pipelines.yml

@@ -45,7 +45,7 @@ jobs:
       Mito27Debian_27:
         python.version: '2.7'
         MODE: mitogen
-        DISTRO: debian
+        DISTRO: debian9
 
       #MitoPy27CentOS6_26:
         #python.version: '2.7'
@@ -60,12 +60,12 @@ jobs:
       Mito37Debian_27:
         python.version: '3.7'
         MODE: mitogen
-        DISTRO: debian
+        DISTRO: debian9
 
       Mito39Debian_27:
         python.version: '3.9'
         MODE: mitogen
-        DISTRO: debian
+        DISTRO: debian9
         VER: 2.10.0
 
       #Py26CentOS7:

tests/ansible/integration/become/sudo_nonexistent.yml

@@ -12,8 +12,10 @@
 
     - name: Verify raw module output.
       assert:
-        that: |
-          out.failed and (
-            ('sudo: unknown user: slartibartfast' in out.msg) or
-            ('sudo: unknown user: slartibartfast' in out.module_stderr)
-          )
+        that:
+          - out.failed
+          # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release)
+          # removed user/group error messages, as defence against CVE-2019-14287.
+          - >-
+            ('sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg))
+            or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10')

tests/ansible/integration/runner/custom_binary_single_null.yml

@@ -15,10 +15,18 @@
       - "out.failed"
       - "out.results[0].failed"
       - "out.results[0].msg.startswith('MODULE FAILURE')"
-      - "out.results[0].module_stdout.startswith('/bin/sh: ')"
+      # On Ubuntu 16.04 /bin/sh is dash 0.5.8. It treats custom_binary_single_null
+      # as a valid executable. There's no error message, and rc == 0.
       - |
-        out.results[0].module_stdout.endswith('custom_binary_single_null: cannot execute binary file\r\n') or
-        out.results[0].module_stdout.endswith('custom_binary_single_null: Exec format error\r\n')
+        out.results[0].module_stdout.startswith('/bin/sh: ')
+        or (ansible_facts.distribution == 'Ubuntu' and ansible_facts.distribution_version == '16.04')
+      - |
+        out.results[0].module_stdout.endswith((
+          'custom_binary_single_null: cannot execute binary file\r\n',
+          'custom_binary_single_null: Exec format error\r\n',
+          'custom_binary_single_null: cannot execute binary file: Exec format error\r\n',
+        ))
+        or (ansible_facts.distribution == 'Ubuntu' and ansible_facts.distribution_version == '16.04')
 
 
 # Can't test this: Mitogen returns 126, 2.5.x returns 126, 2.4.x discarded the

tests/image_prep/README.md

@@ -14,10 +14,11 @@ See ../README.md for a (mostly) description of the accounts created.
 
 ## Building the containers
 
-``./build_docker_images.sh``
-
-Requires Ansible 2.3.x.x in order to target CentOS 5
+No single version of Ansible supports every Linux distribution that we target.
+To workaround this [Tox](https://tox.readthedocs.io) is used, to install and
+run multiple versions of Ansible, in Python virtualenvs.
 
+``tox``
 
 ## Preparing an OS X box
 

tests/image_prep/_container_create.yml

@@ -0,0 +1,20 @@
+- name: Start containers
+  hosts: all
+  strategy: mitogen_free
+  gather_facts: false
+  tasks:
+    - name: Fetch container images
+      docker_image:
+        name: "{{ docker_base }}"
+      delegate_to: localhost
+
+    - name: Start containers
+      docker_container:
+        name: "{{ inventory_hostname }}"
+        image: "{{ docker_base }}"
+        command: /bin/bash
+        hostname: "mitogen-{{ inventory_hostname }}"
+        detach: true
+        interactive: true
+        tty: true
+      delegate_to: localhost

tests/image_prep/_container_finalize.yml

@@ -0,0 +1,18 @@
+- name: Prepare images
+  hosts: all
+  strategy: mitogen_free
+  gather_facts: true
+  tasks:
+    - name: Commit containers
+      command: >
+        docker commit
+        --change 'EXPOSE 22'
+        --change 'CMD ["/usr/sbin/sshd", "-D"]'
+        {{ inventory_hostname }}
+        public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test
+      delegate_to: localhost
+
+    - name: Stop containers
+      command: >
+        docker rm -f {{ inventory_hostname }}
+      delegate_to: localhost

tests/image_prep/_container_setup.yml

@@ -1,95 +1,88 @@
 
 - hosts: all
-  vars_files:
-  - shared_vars.yml
   strategy: linear
   gather_facts: false
   tasks:
-    - raw: >
-        if ! python -c ''; then
-          if type -p yum; then
-            yum -y install python;
-          else
-            apt-get -y update && apt-get -y install python;
-          fi;
+    - name: Install bootstrap packages
+      raw: |
+        set -o errexit
+        set -o nounset
+        if type -p yum; then
+          yum -y install {{ bootstrap_packages | join(' ') }}
+        else
+          apt-get -y update
+          apt-get -y --no-install-recommends install {{ bootstrap_packages | join(' ') }}
         fi
+      when: bootstrap_packages | length
 
 - hosts: all
-  vars_files:
-  - shared_vars.yml
   strategy: mitogen_free
+  # Resource limitation, my laptop freezes doing every container concurrently
+  serial: 4
   # Can't gather facts before here.
   gather_facts: true
   vars:
     distro: "{{ansible_distribution}}"
-    ver: "{{ansible_distribution_major_version}}"
-
-    packages:
-      common:
-        - openssh-server
-        - rsync
-        - strace
-        - sudo
-      Debian:
-        "9":
-          - libjson-perl
-          - python-virtualenv
-          - locales
-      CentOS:
-        "5":
-        - perl
-        - sudo
-          #- perl-JSON  -- skipped on CentOS 5, packages are a pain.
-        "6":
-          - perl-JSON
-        "7":
-          - perl-JSON
-          - python-virtualenv
-
   tasks:
     - when: ansible_virtualization_type != "docker"
       meta: end_play
 
-    - name: Ensure requisite Debian packages are installed
+    - name: Ensure requisite apt packages are installed
       apt:
-        name: "{{packages.common + packages[distro][ver]}}"
-        state: installed
+        name: "{{ common_packages + packages }}"
+        state: present
+        install_recommends: false
         update_cache: true
-      when: distro == "Debian"
+      when: ansible_pkg_mgr == 'apt'
 
-    - name: Ensure requisite Red Hat packaed are installed
+    - name: Ensure requisite yum packages are installed
       yum:
-        name: "{{packages.common + packages[distro][ver]}}"
-        state: installed
+        name: "{{ common_packages + packages }}"
+        state: present
         update_cache: true
-      when: distro == "CentOS"
+      when: ansible_pkg_mgr == 'yum'
 
-    - name: Clean up apt cache
-      command: apt-get clean
-      when: distro == "Debian"
+    - name: Ensure requisite dnf packages are installed
+      dnf:
+        name: "{{ common_packages + packages }}"
+        state: present
+        update_cache: true
+      when: ansible_pkg_mgr == 'dnf'
+
+    - name: Clean up package cache
+      vars:
+        clean_command:
+          apt: apt-get clean
+          yum: yum clean all
+          dnf: dnf clean all
+      command: "{{ clean_command[ansible_pkg_mgr] }}"
+      args:
+        warn: false
 
     - name: Clean up apt package lists
       shell: rm -rf {{item}}/*
       with_items:
       - /var/cache/apt
       - /var/lib/apt/lists
-      when: distro == "Debian"
+      when: ansible_pkg_mgr == 'apt'
 
-    - name: Clean up yum cache
-      command: yum clean all
-      when: distro == "CentOS"
+    - name: Configure /usr/bin/python
+      command: alternatives --set python /usr/bin/python3.8
+      args:
+        creates: /usr/bin/python
+      when: inventory_hostname in ["centos8"]
 
     - name: Enable UTF-8 locale on Debian
       copy:
         dest: /etc/locale.gen
         content: |
           en_US.UTF-8 UTF-8
           fr_FR.UTF-8 UTF-8
-      when: distro == "Debian"
+      when: ansible_pkg_mgr == 'apt'
 
     - name: Generate UTF-8 locale on Debian
       shell: locale-gen
-      when: distro == "Debian"
+      when: ansible_pkg_mgr == 'apt'
 
     - name: Write Unicode into /etc/environment
       copy:
@@ -115,16 +108,6 @@
           permit :mitogen__group
           permit :root
 
-    - name: Vanilla Ansible needs simplejson on CentOS 5.
-      shell: mkdir -p /usr/lib/python2.4/site-packages/simplejson/
-      when: distro == "CentOS" and ver == "5"
-
-    - name: Vanilla Ansible needs simplejson on CentOS 5.
-      synchronize:
-        dest: /usr/lib/python2.4/site-packages/simplejson/
-        src: ../../ansible_mitogen/compat/simplejson/
-      when: distro == "CentOS" and ver == "5"
-
     - name: Set root user password and shell
       user:
         name: root
@@ -182,8 +165,9 @@
     - name: Install CentOS wheel sudo rule
       lineinfile:
         path: /etc/sudoers
-        line: "%wheel ALL=(ALL) ALL"
-      when: distro == "CentOS"
+        regexp: '#* *%wheel +ALL=(ALL) +ALL'
+        line: "%wheel  ALL=(ALL)       ALL"
+      when: ansible_os_family == 'RedHat'
 
     - name: Enable SSH banner
       lineinfile:
@@ -202,6 +186,15 @@
         regexp: '.*session.*required.*pam_loginuid.so'
         line: session optional pam_loginuid.so
 
+    # Normally this would be removed by systemd-networkd-wait-online. If
+    # present ssh works only for root. The message displayed is
+    # > System is booting up. Unprivileged users are not permitted to log in
+    # > yet. Please come back later. For technical details, see pam_nologin(8).
+    - name: Remove login lockout
+      file:
+        path: /run/nologin
+        state: absent
+
     - name: Install convenience script for running an straced Python
       copy:
         mode: 'u+rwx,go=rx'

tests/image_prep/_user_accounts.yml

@@ -5,15 +5,11 @@
 #
 
 - hosts: all
-  vars_files:
-  - shared_vars.yml
   gather_facts: true
   strategy: mitogen_free
   become: true
   vars:
     distro: "{{ansible_distribution}}"
-    ver: "{{ansible_distribution_major_version}}"
-
     special_users:
       - has_sudo
       - has_sudo_nopw
@@ -172,6 +168,8 @@
       with_items:
         - mitogen__pw_required
         - mitogen__require_tty_pw_required
+      when:
+        - ansible_virtualization_type != "docker"
 
     - name: Allow passwordless sudo for require_tty/readonly_homedir
       lineinfile:
@@ -181,10 +179,14 @@
       with_items:
         - mitogen__require_tty
         - mitogen__readonly_homedir
+      when:
+        - ansible_virtualization_type != "docker"
 
     - name: Allow passwordless for many accounts
       lineinfile:
         path: /etc/sudoers
         line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__{{item}}:ALL) NOPASSWD:ALL"
         validate: '/usr/sbin/visudo -cf %s'
       with_items: "{{normal_users}}"
+      when:
+        - ansible_virtualization_type != "docker"

tests/image_prep/ansible.cfg

@@ -1,7 +1,11 @@
 
 [defaults]
+deprecation_warnings = false
 strategy_plugins = ../../ansible_mitogen/plugins/strategy
 retry_files_enabled = false
 display_args_to_stdout = True
 no_target_syslog = True
 host_key_checking = False
+
+[inventory]
+unparsed_is_fatal = true