From c51fb621e7d0156e6f0e89cf38b29f0e1b28f22d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 00:51:34 +0900 Subject: [PATCH 01/21] tests: update to template 1.0.0 --- .ansible-lint | 23 ++++ .flake8 | 4 + .yamllint | 6 + molecule/.gitignore | 1 + molecule/_resources/Dockerfile.j2 | 9 +- .../_resources/{playbook.yml => converge.yml} | 0 molecule/_resources/create-docker.yml | 119 +++++++++++++++--- molecule/_resources/create-lxd.yml | 9 +- molecule/_resources/destroy-docker.yml | 20 ++- molecule/_resources/destroy-lxd.yml | 4 +- molecule/_resources/group_vars/all/common.yml | 37 ++++++ .../group_vars/all/firewall_workaround.yml | 8 ++ molecule/_resources/prepare.yml | 73 +++-------- molecule/_resources/prepare_custom.yml | 47 +++++++ molecule/base_molecule.yml | 43 ++----- molecule/ospo_template_version | 1 + 16 files changed, 290 insertions(+), 114 deletions(-) create mode 100644 .ansible-lint rename molecule/_resources/{playbook.yml => converge.yml} (100%) create mode 100644 molecule/_resources/group_vars/all/common.yml create mode 100644 molecule/_resources/group_vars/all/firewall_workaround.yml create mode 100644 molecule/_resources/prepare_custom.yml create mode 100644 molecule/ospo_template_version diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000..fb18db2 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,23 @@ +--- +exclude_paths: + - .git + - __pycache__ + - .venv + +parseable: true + +# E204: Ansible is not a programming language and using (global) variables +# to make things beautiful may have consequences +# E503: if /.changed/ is used, even if you loop on a registered variable +# and happens to check item.changed, which cannot be converted in to +# a handler, this rule is not clever enough to understand +# E602: https://github.com/ansible/ansible-lint/issues/457 +# https://github.com/ansible/ansible/pull/51030 +skip_list: + - '204' + - '403' + - '405' + - '503' + - '602' + - '702' + diff --git a/.flake8 b/.flake8 index ed113a7..9064e53 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,7 @@ [flake8] ignore = E126,E131,E501,E303,E302,W391 +exclude = + .git, + __pycache__, + .venv diff --git a/.yamllint b/.yamllint index 0e0d60d..3d38986 100644 --- a/.yamllint +++ b/.yamllint @@ -1,5 +1,11 @@ +--- extends: default +ignore: | + .git + __pycache__ + .venv + rules: braces: max-spaces-inside: 1 diff --git a/molecule/.gitignore b/molecule/.gitignore index 8d35cb3..9b94365 100644 --- a/molecule/.gitignore +++ b/molecule/.gitignore @@ -1,2 +1,3 @@ __pycache__ *.pyc +pytestdebug.log diff --git a/molecule/_resources/Dockerfile.j2 b/molecule/_resources/Dockerfile.j2 index b16d569..a4c7921 100644 --- a/molecule/_resources/Dockerfile.j2 +++ b/molecule/_resources/Dockerfile.j2 @@ -6,10 +6,5 @@ FROM {{ item.registry.url }}/{{ item.image }} FROM {{ item.image }} {% endif %} -# on Red Hat systems using dnf, Python 3 is already installed -# the package is versioned and there is no 'python3' dependency package, so better do nothing -RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ - elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python3-devel python3-dnf bash && dnf clean all; \ - elif [ $(command -v yum) ]; then yum makecache fast && yum update -y && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ - elif [ $(command -v zypper) ]; then zypper refresh && zypper update -y && zypper install -y python sudo bash python-xml && zypper clean -a; \ - elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; fi +RUN {{ image_setup }} + diff --git a/molecule/_resources/playbook.yml b/molecule/_resources/converge.yml similarity index 100% rename from molecule/_resources/playbook.yml rename to molecule/_resources/converge.yml diff --git a/molecule/_resources/create-docker.yml b/molecule/_resources/create-docker.yml index bb249be..2b23502 100644 --- a/molecule/_resources/create-docker.yml +++ b/molecule/_resources/create-docker.yml @@ -3,11 +3,14 @@ hosts: localhost connection: local gather_facts: false - no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}" + no_log: "{{ molecule_no_log }}" + vars: + molecule_labels: + owner: molecule tasks: - name: Apply global default parameters set_fact: - final_platforms: "{{ final_platforms | default([]) | union([platform_base.docker|combine(item)]) }}" + final_platforms: "{{ final_platforms | default([]) | union([hostvars[item.name].platform_base.docker|combine(item)]) }}" loop: "{{ molecule_yml.platforms }}" - name: Log into a Docker registry @@ -17,75 +20,157 @@ email: "{{ item.registry.credentials.email | default(omit) }}" registry: "{{ item.registry.url }}" docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}" + cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" with_items: "{{ final_platforms }}" when: - item.registry is defined - item.registry.credentials is defined - item.registry.credentials.username is defined + - name: Check presence of custom Dockerfiles + stat: + path: "{{ molecule_scenario_directory + '/' + (item.dockerfile | default( 'Dockerfile.j2')) }}" + loop: "{{ final_platforms }}" + register: dockerfile_stats + - name: Create Dockerfiles from image names + vars: + # TODO(ssbarnea): expose module dir so it can also be used by plugins + molecule_module_directory: "{{ playbook_dir + '/../../../..' }}" template: - # use a common recipe - src: "{{ molecule_scenario_directory }}/../_resources/Dockerfile.j2" + src: >- + {%- if dockerfile_stats.results[i].stat.exists -%} + {{ molecule_scenario_directory + '/' + (item.dockerfile | default( 'Dockerfile.j2')) }} + {%- else -%} + {{ molecule_module_directory + '/data/Dockerfile.j2' }} + {%- endif -%} dest: "{{ molecule_ephemeral_directory }}/Dockerfile_{{ item.image | regex_replace('[^a-zA-Z0-9_]', '_') }}" - with_items: "{{ final_platforms }}" + loop: "{{ final_platforms }}" + loop_control: + index_var: i when: not item.pre_build_image | default(false) register: platforms - name: Discover local Docker images - docker_image_facts: + docker_image_info: name: "molecule_local/{{ item.item.name }}" docker_host: "{{ item.item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}" + cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" with_items: "{{ platforms.results }}" - when: not item.pre_build_image | default(false) + when: + - not item.pre_build_image | default(false) register: docker_images - - name: Build an Ansible compatible image + - name: Build an Ansible compatible image (new) + when: + - platforms.changed or docker_images.results | map(attribute='images') | select('equalto', []) | list | count >= 0 + - not item.item.pre_build_image | default(false) docker_image: - path: "{{ molecule_ephemeral_directory }}" + build: + path: "{{ molecule_ephemeral_directory }}" + dockerfile: "{{ item.invocation.module_args.dest }}" + pull: "{{ item.item.pull | default(true) }}" + network: "{{ item.item.network_mode | default(omit) }}" + args: "{{ item.item.buildargs | default(omit) }}" name: "molecule_local/{{ item.item.image }}" docker_host: "{{ item.item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}" - dockerfile: "{{ item.item.dockerfile | default(item.invocation.module_args.dest) }}" - force: "{{ item.item.force | default(true) }}" - pull: "{{ item.item.pull | default(omit) }}" + cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" + force_source: "{{ item.item.force | default(true) }}" + source: build with_items: "{{ platforms.results }}" - when: - - platforms.changed or docker_images.results | map(attribute='images') | select('equalto', []) | list | count >= 0 - - not item.item.pre_build_image | default(false) + loop_control: + label: "molecule_local/{{ item.item.image }}" + no_log: false + register: result + until: result is not failed + retries: 3 + delay: 30 - name: Create docker network(s) docker_network: name: "{{ item }}" docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}" + cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" + labels: "{{ molecule_labels | combine(item.labels | default({})) }}" state: present + # Duck: we need IPv6 + enable_ipv6: True + ipam_config: + # RFC 4193 + - subnet: fdd1:0:0:0::/64 with_items: "{{ final_platforms | molecule_get_docker_networks }}" + loop_control: + label: "{{ item }}" + no_log: false + + - name: Determine the CMD directives + set_fact: + command_directives_dict: >- + {{ command_directives_dict | default({}) | + combine({ item.name: item.command | default('bash -c "while true; do sleep 10000; done"') }) + }} + with_items: "{{ final_platforms }}" + when: item.override_command | default(true) - name: Create molecule instance(s) docker_container: name: "{{ item.name }}" docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}" - hostname: "{{ item.hostname | default(item.name) }}" + cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" + hostname: "{{ item.hostname | default(item.name) | replace('@NAME@', item.name) }}" image: "{{ item.pre_build_image | default(false) | ternary('', 'molecule_local/') }}{{ item.image }}" + pull: "{{ item.pull | default(omit) }}" + memory: "{{ item.memory | default(omit) }}" state: started recreate: false log_driver: json-file - command: "{{ item.command | default('bash -c \"while true; do sleep 10000; done\"') }}" + command: "{{ (command_directives_dict | default({}))[item.name] | default(omit) }}" + user: "{{ item.user | default(omit) }}" + pid_mode: "{{ item.pid_mode | default(omit) }}" privileged: "{{ item.privileged | default(omit) }}" security_opts: "{{ item.security_opts | default(omit) }}" + devices: "{{ item.devices | default(omit) }}" volumes: "{{ item.volumes | default(omit) }}" tmpfs: "{{ item.tmpfs | default(omit) }}" capabilities: "{{ item.capabilities | default(omit) }}" + sysctls: "{{ item.sysctls | default(omit) }}" exposed_ports: "{{ item.exposed_ports | default(omit) }}" published_ports: "{{ item.published_ports | default(omit) }}" ulimits: "{{ item.ulimits | default(omit) }}" networks: "{{ item.networks | default(omit) }}" network_mode: "{{ item.network_mode | default(omit) }}" + networks_cli_compatible: "{{ item.networks_cli_compatible | default(true) }}" + purge_networks: "{{ item.purge_networks | default(omit) }}" dns_servers: "{{ item.dns_servers | default(omit) }}" + etc_hosts: "{{ item.etc_hosts | default(omit) }}" env: "{{ item.env | default(omit) }}" restart_policy: "{{ item.restart_policy | default(omit) }}" restart_retries: "{{ item.restart_retries | default(omit) }}" + tty: "{{ item.tty | default(omit) }}" + labels: "{{ molecule_labels | combine(item.labels | default({})) }}" + container_default_behavior: "{{ item.container_default_behavior | default('compatibility' if ansible_version.full is version_compare('2.10', '>=') else omit) }}" + # Duck: missing parameters + dns_search_domains: "{{ item.dns_search_domains | default(omit) }}" register: server with_items: "{{ final_platforms }}" + loop_control: + label: "{{ item.name }}" + no_log: false async: 7200 poll: 0 diff --git a/molecule/_resources/create-lxd.yml b/molecule/_resources/create-lxd.yml index eea8e47..fc83716 100644 --- a/molecule/_resources/create-lxd.yml +++ b/molecule/_resources/create-lxd.yml @@ -7,7 +7,7 @@ tasks: - name: Apply global default parameters set_fact: - final_platforms: "{{ final_platforms | default([]) | union([platform_base.docker|combine(item)]) }}" + final_platforms: "{{ final_platforms | default([]) | union([hostvars[item.name].platform_base.lxd|combine(item)]) }}" loop: "{{ molecule_yml.platforms }}" - name: Create default source variable @@ -33,3 +33,10 @@ wait_for_ipv4_addresses: true timeout: 600 with_items: "{{ final_platforms }}" + +- hosts: all + gather_facts: False + tasks: + - name: "Setup Image" + raw: "{{ image_setup }}" + changed_when: True diff --git a/molecule/_resources/destroy-docker.yml b/molecule/_resources/destroy-docker.yml index 279b24b..2d7b68a 100644 --- a/molecule/_resources/destroy-docker.yml +++ b/molecule/_resources/destroy-docker.yml @@ -3,21 +3,30 @@ hosts: localhost connection: local gather_facts: false - no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}" + no_log: "{{ molecule_no_log }}" tasks: - name: Apply global default parameters set_fact: - final_platforms: "{{ final_platforms | default([]) | union([platform_base.docker|combine(item)]) }}" + final_platforms: "{{ final_platforms | default([]) | union([hostvars[item.name].platform_base.docker|combine(item)]) }}" loop: "{{ molecule_yml.platforms }}" - name: Destroy molecule instance(s) docker_container: name: "{{ item.name }}" docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}" + cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" state: absent force_kill: "{{ item.force_kill | default(true) }}" + keep_volumes: "{{ item.keep_volumes | default(true) }}" + container_default_behavior: "{{ item.container_default_behavior | default('compatibility' if ansible_version.full is version_compare('2.10', '>=') else omit) }}" register: server with_items: "{{ final_platforms }}" + loop_control: + label: "{{ item.name }}" + no_log: false async: 7200 poll: 0 @@ -33,5 +42,12 @@ docker_network: name: "{{ item }}" docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}" + cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" + tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" state: absent with_items: "{{ final_platforms | molecule_get_docker_networks }}" + loop_control: + label: "{{ item }}" + no_log: false diff --git a/molecule/_resources/destroy-lxd.yml b/molecule/_resources/destroy-lxd.yml index 94987d4..0af9c38 100644 --- a/molecule/_resources/destroy-lxd.yml +++ b/molecule/_resources/destroy-lxd.yml @@ -7,12 +7,12 @@ tasks: - name: Apply global default parameters set_fact: - final_platforms: "{{ final_platforms | default([]) | union([platform_base.docker|combine(item)]) }}" + final_platforms: "{{ final_platforms | default([]) | union([hostvars[item.name].platform_base.lxd|combine(item)]) }}" loop: "{{ molecule_yml.platforms }}" - name: Destroy molecule instance(s) lxd_container: - url: "{{ item.url | default(omit)}}" + url: "{{ item.url | default(omit) }}" cert_file: "{{ item.cert_file | default(omit) }}" key_file: "{{ item.key_file | default(omit) }}" trust_password: "{{ item.trust_password | default(omit) }}" diff --git a/molecule/_resources/group_vars/all/common.yml b/molecule/_resources/group_vars/all/common.yml new file mode 100644 index 0000000..418f577 --- /dev/null +++ b/molecule/_resources/group_vars/all/common.yml @@ -0,0 +1,37 @@ +--- +platform_base: + lxd: + source: + type: image + server: https://images.linuxcontainers.org + docker: + dockerfile: ../_resources/Dockerfile.j2 + hostname: "@NAME@.example.com" + command: /sbin/init + privileged: True + network_mode: bridge + networks: + - name: testnetwork + purge_networks: yes + dns_search_domains: + - example.com + +# TODO: readd yum update when Docker cgroup bug is fixed +# TODO: btw why is there no dnf update while there's an equivalent in all other distros? +image_setup: | + if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y systemd-sysv python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python2-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper update -y && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; fi +base_packages: + RedHat: + - epel-release + - net-tools # netstat is needed for testinfra socket check + - bind-utils # the 'dig' utility is needed for tests + - iproute # needed for network facts + Debian: + - net-tools # netstat is needed for testinfra socket check + - dnsutils # the 'dig' utility is needed for tests + - iproute2 # needed for network facts + diff --git a/molecule/_resources/group_vars/all/firewall_workaround.yml b/molecule/_resources/group_vars/all/firewall_workaround.yml new file mode 100644 index 0000000..2daa482 --- /dev/null +++ b/molecule/_resources/group_vars/all/firewall_workaround.yml @@ -0,0 +1,8 @@ +--- +# on Docker starting firewalld after a fresh install breaks the +# network and future yum installation fails with DNS resolution +# problems. I used the Log Denied feature of firewalld to investigate +# but it was empty. At the moment to allow the rest of the role to be +# tested firerwalling management is disabled. +manage_firewall: "{{ ansible_virtualization_type != 'docker' }}" + diff --git a/molecule/_resources/prepare.yml b/molecule/_resources/prepare.yml index c729743..9a31d21 100644 --- a/molecule/_resources/prepare.yml +++ b/molecule/_resources/prepare.yml @@ -2,14 +2,8 @@ - name: Prepare hosts hosts: all + gather_facts: yes tasks: - - name: Install needed tools (Fedora) - package: - name: - # workaround for https://github.com/ansible/ansible/issues/59438 - - hostname - - glibc-all-langpacks - when: ansible_distribution == 'Fedora' or (ansible_os_family == "RedHat" and ansible_distribution_major_version|int >= 8) # to ensure having a real FQDN # (dots are not allowed in LXD container names, thus sticking to short names) - name: set proper hostname @@ -17,54 +11,25 @@ name: "{{ ansible_nodename }}.example.com" # using the `hostname` instantiation parameter for Docker, as /etc/hostname cannot be modified when: ansible_virtualization_type != 'docker' - - name: Install needed tools (EL) + # may cause TLS issues when downloading packages later + - name: Ensure ca-certificates is up-to-date package: - name: - - epel-release - when: ansible_distribution == 'CentOS' or ansible_distribution == 'RedHat' - -- name: Prepare builder host - hosts: ansible-test-builder - tasks: - - name: Install builder tools + name: ca-certificates + state: latest + - name: Install needed tools (RedHat) package: - name: - - sudo # needed for tests - - openssh-clients # to communicate between machines - - crontabs # to setup cron jobs - - name: "Ensure locales variables are preserved with sudo" - lineinfile: - path: /etc/sudoers - line: 'Defaults env_keep += "LANG LANGUAGE LC_ALL"' - state: present - validate: /usr/sbin/visudo -cf %s - # do not interfer with manual tests - - name: Stop cron - service: - name: crond - state: stopped + name: "{{ base_packages[ansible_os_family] }}" + - name: "Prepare Firewall" + block: + - name: "Install Firewalld" + package: + name: firewalld + - name: "Start Firewalld" + service: + name: firewalld + state: started + enabled: yes + when: manage_firewall|default(True) and ansible_os_family == "RedHat" -- name: Prepare web host - hosts: ansible-test-web - tasks: - - name: Install builder tools - package: - name: - - openssh-server # to communicate between machines - - name: start SSH daemon - service: - name: sshd - enabled: yes - state: started - - name: create web builder's user - user: - name: web_builder - comment: "Web Builder User" - - name: create web directory - file: - path: /var/www/www.example.com - state: directory - owner: web_builder - group: web_builder - mode: 0775 +- import_playbook: prepare_custom.yml diff --git a/molecule/_resources/prepare_custom.yml b/molecule/_resources/prepare_custom.yml new file mode 100644 index 0000000..2a052f2 --- /dev/null +++ b/molecule/_resources/prepare_custom.yml @@ -0,0 +1,47 @@ +--- + +- name: Prepare builder host + hosts: ansible-test-builder + tasks: + - name: Install builder tools + package: + name: + - sudo # needed for tests + - openssh-clients # to communicate between machines + - crontabs # to setup cron jobs + - name: "Ensure locales variables are preserved with sudo" + lineinfile: + path: /etc/sudoers + line: 'Defaults env_keep += "LANG LANGUAGE LC_ALL"' + state: present + validate: /usr/sbin/visudo -cf %s + # do not interfer with manual tests + - name: Stop cron + service: + name: crond + state: stopped + +- name: Prepare web host + hosts: ansible-test-web + tasks: + - name: Install builder tools + package: + name: + - openssh-server # to communicate between machines + - name: start SSH daemon + service: + name: sshd + enabled: yes + state: started + - name: create web builder's user + user: + name: web_builder + comment: "Web Builder User" + - name: create web directory + file: + path: /var/www/www.example.com + state: directory + owner: web_builder + group: web_builder + mode: 0775 + diff --git a/molecule/base_molecule.yml b/molecule/base_molecule.yml index d5e3191..2cbc1e5 100644 --- a/molecule/base_molecule.yml +++ b/molecule/base_molecule.yml @@ -2,40 +2,21 @@ dependency: name: galaxy driver: - name: lxd -lint: - name: yamllint + name: docker +lint: | + set -e + yamllint . + ansible-lint --nocolor + flake8 provisioner: name: ansible options: vv: True - inventory: - group_vars: - all: - platform_base: - lxd: - source: - type: image - server: https://images.linuxcontainers.org - docker: - hostname: "{{ item.name }}.example.com" - command: /sbin/init - privileged: True - network_mode: bridge - networks: - - name: testnetwork - aliases: - # allow access via short DNS name - - "{{ item.name }}" - # used to switch the testing branch - tests_is_container: False - host_vars: - ansible-test-builder: - ansible-test-web: + config_options: + defaults: + hash_behaviour: merge env: - LC_ALL: en_US.UTF-8 LANG: en_US.UTF-8 - LANGUAGE: en_US.UTF-8 playbooks: docker: create: ../_resources/create-docker.yml @@ -44,10 +25,10 @@ provisioner: create: ../_resources/create-lxd.yml destroy: ../_resources/destroy-lxd.yml prepare: ../_resources/prepare.yml - converge: ../_resources/playbook.yml - lint: - name: ansible-lint + converge: ../_resources/converge.yml verifier: name: testinfra + options: + v: True lint: name: flake8 diff --git a/molecule/ospo_template_version b/molecule/ospo_template_version new file mode 100644 index 0000000..7dea76e --- /dev/null +++ b/molecule/ospo_template_version @@ -0,0 +1 @@ +1.0.1 From 349dbd10f19ed5fa3b057f9709510bc5cb83c485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 01:01:05 +0900 Subject: [PATCH 02/21] Fix W504 Also ignore W503 since both are reported and conflict. See https://gitlab.com/pycqa/flake8/-/issues/463 --- .flake8 | 2 +- templates/build_deploy.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.flake8 b/.flake8 index 9064e53..a85f44e 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -ignore = E126,E131,E501,E303,E302,W391 +ignore = E126,E131,E501,E303,E302,W391,W503 exclude = .git, __pycache__, diff --git a/templates/build_deploy.py b/templates/build_deploy.py index 5e4ee3a..8e88e58 100644 --- a/templates/build_deploy.py +++ b/templates/build_deploy.py @@ -189,8 +189,8 @@ def do_rsync(source): 'UserKnownHostsFile=/dev/null ' '-o ' 'StrictHostKeyChecking=no ' - '-i ' + - os.path.expanduser('~/.ssh/{}_id.rsa'.format(name)), + '-i ' + + os.path.expanduser('~/.ssh/{}_id.rsa'.format(name)), '--delete-after', '-rltogvz', '--omit-dir-times', @@ -247,9 +247,9 @@ def do_rsync(source): last_build = datetime.datetime.fromtimestamp( int(status.get("last_build", "0"))) -if ('regular_rebuild_interval' in config and - datetime.datetime.now() - last_build > - datetime.timedelta(hours=config['regular_rebuild_interval'])): +if ('regular_rebuild_interval' in config + and datetime.datetime.now() - last_build + > datetime.timedelta(hours=config['regular_rebuild_interval'])): start_build = True current_commit = get_last_commit(checkout_dir) From f5f13410f28a7b30de8d9f0a1ecdde1decb7ac7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 01:13:28 +0900 Subject: [PATCH 03/21] Acknowledge we do not support Ascii Binder anymore The code is still there and should work. --- README.md | 4 +-- molecule/ascii_binder-f30/molecule.yml | 25 ------------------- .../ascii_binder-f30/tests/test_default.py | 1 - 3 files changed, 2 insertions(+), 28 deletions(-) delete mode 100644 molecule/ascii_binder-f30/molecule.yml delete mode 120000 molecule/ascii_binder-f30/tests/test_default.py diff --git a/README.md b/README.md index 256265c..a996a5a 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ Ansible module used by OSAS to deploy a static website builder. While working, the role is still being written, as seen by the TODO list at the end of this document. -Currently this role supports the following web generators: -* Ascii Binder (http://asciibinder.org/) +Currently this role works with the following web generators: +* Ascii Binder (http://asciibinder.org/), should still work but not supported anymore * [Jekyll](https://jekyllrb.com/) * [Middleman](https://middlemanapp.com/) * [Nikola](https://getnikola.com/) diff --git a/molecule/ascii_binder-f30/molecule.yml b/molecule/ascii_binder-f30/molecule.yml deleted file mode 100644 index 3a0ef30..0000000 --- a/molecule/ascii_binder-f30/molecule.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -scenario: - name: ascii_binder_f30 -platforms: - - name: ansible-test-builder - # LXD - source: - alias: 'fedora/30/amd64' - # Docker - image: 'fedora:30' - # this one is always the same - - name: ansible-test-web - # LXD - source: - alias: 'centos/7/amd64' - # Docker - image: 'centos:7' -provisioner: - inventory: - host_vars: - ansible-test-builder: - ansible_python_interpreter: /usr/bin/python3 - builder: ascii_binder - # known working website - git_url: https://github.com/redhataccess/ascii_binder-docs.git diff --git a/molecule/ascii_binder-f30/tests/test_default.py b/molecule/ascii_binder-f30/tests/test_default.py deleted file mode 120000 index 7363278..0000000 --- a/molecule/ascii_binder-f30/tests/test_default.py +++ /dev/null @@ -1 +0,0 @@ -../../_resources/tests/test_default.py \ No newline at end of file From 682b14d688d7daa5da048f5cdcec3d6251175745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 03:12:50 +0900 Subject: [PATCH 04/21] tests: switch to Python 3 for Red Hat systems using dnf --- molecule/_resources/group_vars/all/common.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molecule/_resources/group_vars/all/common.yml b/molecule/_resources/group_vars/all/common.yml index 418f577..7777864 100644 --- a/molecule/_resources/group_vars/all/common.yml +++ b/molecule/_resources/group_vars/all/common.yml @@ -20,7 +20,7 @@ platform_base: # TODO: btw why is there no dnf update while there's an equivalent in all other distros? image_setup: | if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y systemd-sysv python sudo bash ca-certificates && apt-get clean; \ - elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python2-dnf bash && dnf clean all; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python3 sudo python3-devel python3-dnf bash && dnf clean all; \ elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ elif [ $(command -v zypper) ]; then zypper refresh && zypper update -y && zypper install -y python sudo bash python-xml && zypper clean -a; \ elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; fi From 58938c4013645ae03ebed0dc1b0e93d163108b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 03:13:57 +0900 Subject: [PATCH 05/21] add missing gems build dependencies --- vars/Debian.yml | 1 + vars/RedHat.yml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/vars/Debian.yml b/vars/Debian.yml index 22d7191..4cbcb04 100644 --- a/vars/Debian.yml +++ b/vars/Debian.yml @@ -23,6 +23,7 @@ package_list: - libxslt1-dev - libidn11-dev - nodejs + - libssl-dev middleman: [] jekyll: [] ascii_binder: [] diff --git a/vars/RedHat.yml b/vars/RedHat.yml index 0376461..8fb3d27 100644 --- a/vars/RedHat.yml +++ b/vars/RedHat.yml @@ -25,6 +25,9 @@ package_list: - libxslt-devel - libidn-devel - nodejs + - openssl-devel + # needed for /usr/lib/rpm/redhat/redhat-hardened-cc1 + - redhat-rpm-config middleman: [] jekyll: [] ascii_binder: [] From 6b66cbc5d187a210f86be38efc9e318744d72f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Wed, 8 Jul 2020 23:29:47 +0900 Subject: [PATCH 06/21] fix bug with ternary + undefined variable --- tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/main.yml b/tasks/main.yml index bc4ff74..26e88d7 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -141,9 +141,9 @@ - name: "Schedule build and sync" include_tasks: schedule_build_and_sync.yml - when: (tests_is_container is defined) | ternary(not tests_is_container, ansible_env.container is undefined) + when: ((not tests_is_container) if tests_is_container is defined else (ansible_env.container is undefined)) - name: "Build now (container)" include_tasks: build.yml - when: ((tests_is_container is defined) | ternary(tests_is_container, ansible_env.container is defined)) and builder_container_build_now|bool + when: (tests_is_container if tests_is_container is defined else (ansible_env.container is defined)) and builder_container_build_now|bool From 35f7c5a4d95c7c150d368fe8230416fd00742119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 19:13:16 +0900 Subject: [PATCH 07/21] tests: don't test idempotence on in-place build (container case) --- tasks/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks/build.yml b/tasks/build.yml index 37c16ff..29380cd 100644 --- a/tasks/build.yml +++ b/tasks/build.yml @@ -9,4 +9,5 @@ become_user: "{{ builder_username }}" become: True changed_when: True + tags: molecule-idempotence-notest From c82d3d021ebd7c48cd67bdcc2da5a61e2be46a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 19:14:07 +0900 Subject: [PATCH 08/21] tests: properly set tests_is_container variable for each case --- molecule/_resources/converge.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/molecule/_resources/converge.yml b/molecule/_resources/converge.yml index 5c051b7..cd638c5 100644 --- a/molecule/_resources/converge.yml +++ b/molecule/_resources/converge.yml @@ -13,10 +13,12 @@ rsync_server: ansible-test-web rsync_user: web_builder rsync_location: /var/www/www.example.com + tests_is_container: False - name: "Testing role (container)" include_role: name: ansible-role-web_builder vars: builder_name: testbuilder_container builder_username: web_builder + tests_is_container: True From 13c6aee38895b29542188cf8a8d64a42e4a12471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 19:16:02 +0900 Subject: [PATCH 09/21] tests: ensure cron cannot be restarted This conflicts with running the website build either in containers (in-place) or in the tests. --- molecule/_resources/prepare_custom.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/molecule/_resources/prepare_custom.yml b/molecule/_resources/prepare_custom.yml index 2a052f2..9a22a58 100644 --- a/molecule/_resources/prepare_custom.yml +++ b/molecule/_resources/prepare_custom.yml @@ -19,6 +19,7 @@ - name: Stop cron service: name: crond + enabled: no state: stopped - name: Prepare web host From 4824b4354df4f033575e7684bcfd23abfa71452d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 19:17:13 +0900 Subject: [PATCH 10/21] tests: open SSH port for rsync case --- molecule/_resources/prepare_custom.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/molecule/_resources/prepare_custom.yml b/molecule/_resources/prepare_custom.yml index 9a22a58..296228d 100644 --- a/molecule/_resources/prepare_custom.yml +++ b/molecule/_resources/prepare_custom.yml @@ -34,6 +34,13 @@ name: sshd enabled: yes state: started + - name: Open the firewall for {{ ssh_port }} using firewalld + firewalld: + port: "22/tcp" + permanent: true + state: enabled + immediate: yes + when: manage_firewall|default(True) and ansible_os_family == "RedHat" - name: create web builder's user user: name: web_builder From 1153abe5d37d1b11a957fa54a3b84904b990b108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 19:18:50 +0900 Subject: [PATCH 11/21] tests: work around Ansible#70168 Set ansible_python_interpreter in all scenario to avoid the bug. --- molecule/jekyll-f30/molecule.yml | 5 ++++- molecule/middleman-el7/molecule.yml | 4 ++++ molecule/nikola-el8/molecule.yml | 4 ++++ molecule/planet-el7/molecule.yml | 4 ++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/molecule/jekyll-f30/molecule.yml b/molecule/jekyll-f30/molecule.yml index 6e45179..26d4512 100644 --- a/molecule/jekyll-f30/molecule.yml +++ b/molecule/jekyll-f30/molecule.yml @@ -17,9 +17,12 @@ platforms: image: 'centos:7' provisioner: inventory: + group_vars: + all: + # workaround for https://github.com/ansible/ansible/issues/70168 + ansible_python_interpreter: /usr/bin/python3 host_vars: ansible-test-builder: - ansible_python_interpreter: /usr/bin/python3 builder: jekyll # known working website git_url: https://gitlab.com/osas/osci.io.git diff --git a/molecule/middleman-el7/molecule.yml b/molecule/middleman-el7/molecule.yml index 9e2f6ad..400bcf6 100644 --- a/molecule/middleman-el7/molecule.yml +++ b/molecule/middleman-el7/molecule.yml @@ -17,6 +17,10 @@ platforms: image: 'centos:7' provisioner: inventory: + group_vars: + all: + # workaround for https://github.com/ansible/ansible/issues/70168 + ansible_python_interpreter: /usr/bin/python host_vars: ansible-test-builder: builder: middleman diff --git a/molecule/nikola-el8/molecule.yml b/molecule/nikola-el8/molecule.yml index 215122d..da01be7 100644 --- a/molecule/nikola-el8/molecule.yml +++ b/molecule/nikola-el8/molecule.yml @@ -17,6 +17,10 @@ platforms: image: 'centos:8' provisioner: inventory: + group_vars: + all: + # workaround for https://github.com/ansible/ansible/issues/70168 + ansible_python_interpreter: /usr/bin/python3 host_vars: ansible-test-builder: builder: nikola diff --git a/molecule/planet-el7/molecule.yml b/molecule/planet-el7/molecule.yml index 430750e..f915a22 100644 --- a/molecule/planet-el7/molecule.yml +++ b/molecule/planet-el7/molecule.yml @@ -17,6 +17,10 @@ platforms: image: 'centos:7' provisioner: inventory: + group_vars: + all: + # workaround for https://github.com/ansible/ansible/issues/70168 + ansible_python_interpreter: /usr/bin/python host_vars: ansible-test-builder: builder: planet From 2b84076f240be3695a483a330458f219f9e34636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 19:21:16 +0900 Subject: [PATCH 12/21] tests: fix hostname domain Don't add 'example.com' domain if already present. This situation occurs when running again a test without destroying the molecule containers. --- molecule/_resources/prepare.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molecule/_resources/prepare.yml b/molecule/_resources/prepare.yml index 9a31d21..3544101 100644 --- a/molecule/_resources/prepare.yml +++ b/molecule/_resources/prepare.yml @@ -10,7 +10,7 @@ hostname: name: "{{ ansible_nodename }}.example.com" # using the `hostname` instantiation parameter for Docker, as /etc/hostname cannot be modified - when: ansible_virtualization_type != 'docker' + when: ansible_virtualization_type != 'docker' and 'example.com' not in ansible_fqdn # may cause TLS issues when downloading packages later - name: Ensure ca-certificates is up-to-date package: From 13c5fff99ef53a5371129c10fc81452475e4165f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 21:49:22 +0900 Subject: [PATCH 13/21] tests: update docker templates for Molecule 3.0.5 --- molecule/_resources/create-docker.yml | 72 ++++--------------- molecule/_resources/destroy-docker.yml | 19 ++--- molecule/_resources/group_vars/all/common.yml | 7 ++ 3 files changed, 26 insertions(+), 72 deletions(-) diff --git a/molecule/_resources/create-docker.yml b/molecule/_resources/create-docker.yml index 2b23502..04b242b 100644 --- a/molecule/_resources/create-docker.yml +++ b/molecule/_resources/create-docker.yml @@ -4,9 +4,6 @@ connection: local gather_facts: false no_log: "{{ molecule_no_log }}" - vars: - molecule_labels: - owner: molecule tasks: - name: Apply global default parameters set_fact: @@ -30,29 +27,14 @@ - item.registry.credentials is defined - item.registry.credentials.username is defined - - name: Check presence of custom Dockerfiles - stat: - path: "{{ molecule_scenario_directory + '/' + (item.dockerfile | default( 'Dockerfile.j2')) }}" - loop: "{{ final_platforms }}" - register: dockerfile_stats - - name: Create Dockerfiles from image names - vars: - # TODO(ssbarnea): expose module dir so it can also be used by plugins - molecule_module_directory: "{{ playbook_dir + '/../../../..' }}" template: - src: >- - {%- if dockerfile_stats.results[i].stat.exists -%} - {{ molecule_scenario_directory + '/' + (item.dockerfile | default( 'Dockerfile.j2')) }} - {%- else -%} - {{ molecule_module_directory + '/data/Dockerfile.j2' }} - {%- endif -%} + src: "{{ item.dockerfile | default('Dockerfile.j2') }}" dest: "{{ molecule_ephemeral_directory }}/Dockerfile_{{ item.image | regex_replace('[^a-zA-Z0-9_]', '_') }}" - loop: "{{ final_platforms }}" - loop_control: - index_var: i + with_items: "{{ final_platforms }}" when: not item.pre_build_image | default(false) register: platforms + no_log: item.failed - name: Discover local Docker images docker_image_info: @@ -63,11 +45,10 @@ key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" with_items: "{{ platforms.results }}" - when: - - not item.pre_build_image | default(false) + when: not item.pre_build_image | default(false) register: docker_images - - name: Build an Ansible compatible image (new) + - name: Build an Ansible compatible image when: - platforms.changed or docker_images.results | map(attribute='images') | select('equalto', []) | list | count >= 0 - not item.item.pre_build_image | default(false) @@ -76,8 +57,6 @@ path: "{{ molecule_ephemeral_directory }}" dockerfile: "{{ item.invocation.module_args.dest }}" pull: "{{ item.item.pull | default(true) }}" - network: "{{ item.item.network_mode | default(omit) }}" - args: "{{ item.item.buildargs | default(omit) }}" name: "molecule_local/{{ item.item.image }}" docker_host: "{{ item.item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}" cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" @@ -86,6 +65,7 @@ tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" force_source: "{{ item.item.force | default(true) }}" source: build + buildargs: "{{ item.item.buildargs | default(omit) }}" with_items: "{{ platforms.results }}" loop_control: label: "molecule_local/{{ item.item.image }}" @@ -96,24 +76,9 @@ delay: 30 - name: Create docker network(s) - docker_network: - name: "{{ item }}" - docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}" - cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" - cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" - key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" - tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" - labels: "{{ molecule_labels | combine(item.labels | default({})) }}" - state: present - # Duck: we need IPv6 - enable_ipv6: True - ipam_config: - # RFC 4193 - - subnet: fdd1:0:0:0::/64 - with_items: "{{ final_platforms | molecule_get_docker_networks }}" - loop_control: - label: "{{ item }}" - no_log: false + action: docker_network + args: "{{ docker_network_defaults | default({}) | combine(item) }}" + with_items: "{{ final_platforms | molecule_get_docker_networks('present') }}" - name: Determine the CMD directives set_fact: @@ -132,45 +97,37 @@ cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" - hostname: "{{ item.hostname | default(item.name) | replace('@NAME@', item.name) }}" + hostname: "{{ item.hostname | default(item.name) }}" image: "{{ item.pre_build_image | default(false) | ternary('', 'molecule_local/') }}{{ item.image }}" - pull: "{{ item.pull | default(omit) }}" + pull: "{{ item.pull | default (omit) }}" memory: "{{ item.memory | default(omit) }}" + memory_swap: "{{ item.memory_swap | default(omit) }}" state: started recreate: false log_driver: json-file - command: "{{ (command_directives_dict | default({}))[item.name] | default(omit) }}" user: "{{ item.user | default(omit) }}" + command: "{{ (command_directives_dict | default({}))[item.name] | default(omit) }}" pid_mode: "{{ item.pid_mode | default(omit) }}" privileged: "{{ item.privileged | default(omit) }}" security_opts: "{{ item.security_opts | default(omit) }}" - devices: "{{ item.devices | default(omit) }}" volumes: "{{ item.volumes | default(omit) }}" tmpfs: "{{ item.tmpfs | default(omit) }}" capabilities: "{{ item.capabilities | default(omit) }}" - sysctls: "{{ item.sysctls | default(omit) }}" exposed_ports: "{{ item.exposed_ports | default(omit) }}" published_ports: "{{ item.published_ports | default(omit) }}" ulimits: "{{ item.ulimits | default(omit) }}" networks: "{{ item.networks | default(omit) }}" - network_mode: "{{ item.network_mode | default(omit) }}" networks_cli_compatible: "{{ item.networks_cli_compatible | default(true) }}" + network_mode: "{{ item.network_mode | default(omit) }}" purge_networks: "{{ item.purge_networks | default(omit) }}" dns_servers: "{{ item.dns_servers | default(omit) }}" etc_hosts: "{{ item.etc_hosts | default(omit) }}" env: "{{ item.env | default(omit) }}" restart_policy: "{{ item.restart_policy | default(omit) }}" restart_retries: "{{ item.restart_retries | default(omit) }}" - tty: "{{ item.tty | default(omit) }}" - labels: "{{ molecule_labels | combine(item.labels | default({})) }}" container_default_behavior: "{{ item.container_default_behavior | default('compatibility' if ansible_version.full is version_compare('2.10', '>=') else omit) }}" - # Duck: missing parameters - dns_search_domains: "{{ item.dns_search_domains | default(omit) }}" register: server with_items: "{{ final_platforms }}" - loop_control: - label: "{{ item.name }}" - no_log: false async: 7200 poll: 0 @@ -181,3 +138,4 @@ until: docker_jobs.finished retries: 300 with_items: "{{ server.results }}" + no_log: item.failed diff --git a/molecule/_resources/destroy-docker.yml b/molecule/_resources/destroy-docker.yml index 2d7b68a..0a1733a 100644 --- a/molecule/_resources/destroy-docker.yml +++ b/molecule/_resources/destroy-docker.yml @@ -24,9 +24,6 @@ container_default_behavior: "{{ item.container_default_behavior | default('compatibility' if ansible_version.full is version_compare('2.10', '>=') else omit) }}" register: server with_items: "{{ final_platforms }}" - loop_control: - label: "{{ item.name }}" - no_log: false async: 7200 poll: 0 @@ -37,17 +34,9 @@ until: docker_jobs.finished retries: 300 with_items: "{{ server.results }}" + no_log: item.failed - name: Delete docker network(s) - docker_network: - name: "{{ item }}" - docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}" - cacert_path: "{{ item.cacert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/ca.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" - cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" - key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" - tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" - state: absent - with_items: "{{ final_platforms | molecule_get_docker_networks }}" - loop_control: - label: "{{ item }}" - no_log: false + action: docker_network + args: "{{ docker_network_defaults | default({}) | combine(item) }}" + with_items: "{{ final_platforms | molecule_get_docker_networks('absent') }}" diff --git a/molecule/_resources/group_vars/all/common.yml b/molecule/_resources/group_vars/all/common.yml index 7777864..f41a5d2 100644 --- a/molecule/_resources/group_vars/all/common.yml +++ b/molecule/_resources/group_vars/all/common.yml @@ -16,6 +16,13 @@ platform_base: dns_search_domains: - example.com +docker_network_defaults: + # Duck: we need IPv6 + enable_ipv6: True + ipam_config: + # RFC 4193 + - subnet: fdd1:0:0:0::/64 + # TODO: readd yum update when Docker cgroup bug is fixed # TODO: btw why is there no dnf update while there's an equivalent in all other distros? image_setup: | From 96891ddac18d32a28eb6a5abfc49302ced406f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Thu, 9 Jul 2020 23:13:36 +0900 Subject: [PATCH 14/21] tests: install linters for shippable --- shippable.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/shippable.yml b/shippable.yml index fecfd2a..29934ff 100644 --- a/shippable.yml +++ b/shippable.yml @@ -20,6 +20,7 @@ build: - docker info - pip3 install molecule - pip3 install docker + - pip3 install yamllint ansible-lint flake8 - molecule --version - molecule --debug --base-config molecule/base_molecule.yml test --driver-name docker --all From 7662f62b4726928234d04fe55cadd28fc73f5a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Fri, 10 Jul 2020 00:03:44 +0900 Subject: [PATCH 15/21] tests: switch Jekyll tests to EL8 Keep web host on a specific EL7 version to workaround a Docker bug. --- .../{jekyll-f30 => jekyll-el8}/molecule.yml | 21 ++++++++++++------- .../tests/test_default.py | 0 2 files changed, 13 insertions(+), 8 deletions(-) rename molecule/{jekyll-f30 => jekyll-el8}/molecule.yml (56%) rename molecule/{jekyll-f30 => jekyll-el8}/tests/test_default.py (100%) diff --git a/molecule/jekyll-f30/molecule.yml b/molecule/jekyll-el8/molecule.yml similarity index 56% rename from molecule/jekyll-f30/molecule.yml rename to molecule/jekyll-el8/molecule.yml index 26d4512..20fe34b 100644 --- a/molecule/jekyll-f30/molecule.yml +++ b/molecule/jekyll-el8/molecule.yml @@ -1,28 +1,33 @@ --- scenario: - name: jekyll-f30 + name: jekyll-el8 +# pin centos version to workaround a bug in Docker +# https://github.com/docker/for-linux/issues/835 +# https://github.com/systemd/systemd/issues/11752 platforms: - name: ansible-test-builder # LXD source: - alias: 'fedora/30/amd64' + alias: 'centos/8/amd64' # Docker - image: 'fedora:30' + image: 'centos:8' # this one is always the same - name: ansible-test-web # LXD source: alias: 'centos/7/amd64' # Docker - image: 'centos:7' + image: 'centos:7.6.1810' provisioner: inventory: - group_vars: - all: - # workaround for https://github.com/ansible/ansible/issues/70168 - ansible_python_interpreter: /usr/bin/python3 host_vars: ansible-test-builder: builder: jekyll # known working website git_url: https://gitlab.com/osas/osci.io.git + # workaround for https://github.com/ansible/ansible/issues/70168 + ansible_python_interpreter: /usr/bin/python3 + ansible-test-web: + # workaround for https://github.com/ansible/ansible/issues/70168 + ansible_python_interpreter: /usr/bin/python + diff --git a/molecule/jekyll-f30/tests/test_default.py b/molecule/jekyll-el8/tests/test_default.py similarity index 100% rename from molecule/jekyll-f30/tests/test_default.py rename to molecule/jekyll-el8/tests/test_default.py From 879824a5f54e98178781fd01e4e274438503fe17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Fri, 10 Jul 2020 00:06:32 +0900 Subject: [PATCH 16/21] tests: pin EL7 for nikola-el8 scenario to workaround a Docker bug --- molecule/nikola-el8/molecule.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/molecule/nikola-el8/molecule.yml b/molecule/nikola-el8/molecule.yml index da01be7..d1d17ce 100644 --- a/molecule/nikola-el8/molecule.yml +++ b/molecule/nikola-el8/molecule.yml @@ -1,6 +1,9 @@ --- scenario: name: nikola-el8 +# pin centos version to workaround a bug in Docker +# https://github.com/docker/for-linux/issues/835 +# https://github.com/systemd/systemd/issues/11752 platforms: - name: ansible-test-builder # LXD @@ -12,17 +15,19 @@ platforms: - name: ansible-test-web # LXD source: - alias: 'centos/8/amd64' + alias: 'centos/7/amd64' # Docker - image: 'centos:8' + image: 'centos:7.6.1810' provisioner: inventory: - group_vars: - all: - # workaround for https://github.com/ansible/ansible/issues/70168 - ansible_python_interpreter: /usr/bin/python3 host_vars: ansible-test-builder: builder: nikola # known working website git_url: https://pagure.io/koji-site.git + # workaround for https://github.com/ansible/ansible/issues/70168 + ansible_python_interpreter: /usr/bin/python3 + ansible-test-web: + # workaround for https://github.com/ansible/ansible/issues/70168 + ansible_python_interpreter: /usr/bin/python + From dcd4cd0c6f7807e17717d9882730cc4a323c2ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Fri, 10 Jul 2020 00:33:02 +0900 Subject: [PATCH 17/21] tests: pin EL7 for middleman-el7 scenario to workaorund a Docker bug --- molecule/middleman-el7/molecule.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molecule/middleman-el7/molecule.yml b/molecule/middleman-el7/molecule.yml index 400bcf6..403251e 100644 --- a/molecule/middleman-el7/molecule.yml +++ b/molecule/middleman-el7/molecule.yml @@ -14,7 +14,7 @@ platforms: source: alias: 'centos/7/amd64' # Docker - image: 'centos:7' + image: 'centos:7.6.1810' provisioner: inventory: group_vars: From 2c6955e1f8eb04d7a5c14c913ecc88611778f144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Fri, 10 Jul 2020 18:38:20 +0900 Subject: [PATCH 18/21] tests: fix docker container hostname --- molecule/_resources/create-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molecule/_resources/create-docker.yml b/molecule/_resources/create-docker.yml index 04b242b..e4f5315 100644 --- a/molecule/_resources/create-docker.yml +++ b/molecule/_resources/create-docker.yml @@ -97,7 +97,7 @@ cert_path: "{{ item.cert_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/cert.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" key_path: "{{ item.key_path | default((lookup('env', 'DOCKER_CERT_PATH') + '/key.pem') if lookup('env', 'DOCKER_CERT_PATH') else omit) }}" tls_verify: "{{ item.tls_verify | default(lookup('env', 'DOCKER_TLS_VERIFY')) or false }}" - hostname: "{{ item.hostname | default(item.name) }}" + hostname: "{{ item.hostname | default(item.name) | replace('@NAME@', item.name) }}" image: "{{ item.pre_build_image | default(false) | ternary('', 'molecule_local/') }}{{ item.image }}" pull: "{{ item.pull | default (omit) }}" memory: "{{ item.memory | default(omit) }}" From e1fc31008762aee37661df5b39a14c94b9cc2455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Fri, 10 Jul 2020 19:49:11 +0900 Subject: [PATCH 19/21] tests: more debug info --- molecule/_resources/prepare.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/molecule/_resources/prepare.yml b/molecule/_resources/prepare.yml index 3544101..75dc1a3 100644 --- a/molecule/_resources/prepare.yml +++ b/molecule/_resources/prepare.yml @@ -4,6 +4,18 @@ hosts: all gather_facts: yes tasks: + - name: "Display kernel version" + debug: + var: ansible_kernel_version + - name: "Display OS version #1" + slurp: + src: "{{ (ansible_os_family == 'Debian') | ternary('/etc/debian_version', '/etc/redhat-release') }}" + ignore_errors: True + changed_when: False + register: os_version + - name: "Display OS version #1" + debug: + msg: "{{ os_version['content'] | b64decode }}" # to ensure having a real FQDN # (dots are not allowed in LXD container names, thus sticking to short names) - name: set proper hostname From c55a09eb74726ed4c3c2f7a46c4865b425b501d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Tue, 8 Sep 2020 03:38:45 +0900 Subject: [PATCH 20/21] fix E208 --- tasks/main.yml | 4 ++++ tasks/schedule_build_and_sync.yml | 1 + 2 files changed, 5 insertions(+) diff --git a/tasks/main.yml b/tasks/main.yml index 26e88d7..60c92f6 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -70,6 +70,7 @@ path: "/srv/builder/{{ builder_repo[builder].subdest }}/{{ item }}" owner: '{{ builder_username }}' group: '{{ builder_username }}' + mode: 0755 recurse: yes with_items: "{{ builder_repo[builder].writable_subdirs | default([]) }}" when: builder_repo[builder] is defined @@ -132,6 +133,9 @@ template: dest: "/srv/builder/{{ builder_name }}.yml" src: builder.yml.j2 + owner: '{{ builder_username }}' + group: '{{ builder_username }}' + mode: 0644 - name: Install Rsync package: diff --git a/tasks/schedule_build_and_sync.yml b/tasks/schedule_build_and_sync.yml index 513c2f2..9616697 100644 --- a/tasks/schedule_build_and_sync.yml +++ b/tasks/schedule_build_and_sync.yml @@ -53,6 +53,7 @@ path: /srv/builder/.ssh/config owner: "{{ builder_username }}" group: "{{ builder_username }}" + mode: 0644 block: | Match Host {{ rsync_server }} User {{ rsync_user }} IdentityFile /srv/builder/.ssh/{{ builder_name }}_id.rsa From 3614aad8638d7c720232dd8c619f254a8cf7d57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Dequ=C3=A8nes=20=28Duck=29?= Date: Tue, 8 Sep 2020 03:39:25 +0900 Subject: [PATCH 21/21] tests: update template to 1.0.1 --- molecule/_resources/create-docker.yml | 1 + molecule/_resources/group_vars/all/common.yml | 2 +- molecule/_resources/prepare.yml | 12 ------------ 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/molecule/_resources/create-docker.yml b/molecule/_resources/create-docker.yml index e4f5315..4e84eac 100644 --- a/molecule/_resources/create-docker.yml +++ b/molecule/_resources/create-docker.yml @@ -28,6 +28,7 @@ - item.registry.credentials.username is defined - name: Create Dockerfiles from image names + # noqa 208 template: src: "{{ item.dockerfile | default('Dockerfile.j2') }}" dest: "{{ molecule_ephemeral_directory }}/Dockerfile_{{ item.image | regex_replace('[^a-zA-Z0-9_]', '_') }}" diff --git a/molecule/_resources/group_vars/all/common.yml b/molecule/_resources/group_vars/all/common.yml index f41a5d2..7725c78 100644 --- a/molecule/_resources/group_vars/all/common.yml +++ b/molecule/_resources/group_vars/all/common.yml @@ -26,7 +26,7 @@ docker_network_defaults: # TODO: readd yum update when Docker cgroup bug is fixed # TODO: btw why is there no dnf update while there's an equivalent in all other distros? image_setup: | - if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y systemd-sysv python sudo bash ca-certificates && apt-get clean; \ + if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y systemd-sysv python3 sudo bash ca-certificates && apt-get clean; \ elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python3 sudo python3-devel python3-dnf bash && dnf clean all; \ elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ elif [ $(command -v zypper) ]; then zypper refresh && zypper update -y && zypper install -y python sudo bash python-xml && zypper clean -a; \ diff --git a/molecule/_resources/prepare.yml b/molecule/_resources/prepare.yml index 75dc1a3..3544101 100644 --- a/molecule/_resources/prepare.yml +++ b/molecule/_resources/prepare.yml @@ -4,18 +4,6 @@ hosts: all gather_facts: yes tasks: - - name: "Display kernel version" - debug: - var: ansible_kernel_version - - name: "Display OS version #1" - slurp: - src: "{{ (ansible_os_family == 'Debian') | ternary('/etc/debian_version', '/etc/redhat-release') }}" - ignore_errors: True - changed_when: False - register: os_version - - name: "Display OS version #1" - debug: - msg: "{{ os_version['content'] | b64decode }}" # to ensure having a real FQDN # (dots are not allowed in LXD container names, thus sticking to short names) - name: set proper hostname