tests: Add a playbook and job to run sanity tests

This is meant to install every collection in a release's
galaxy-requirements.yaml and then iterate over them to run sanity tests
on each.

Note that errors are not fatal yet (ignore_errors: true).

Co-authored-by: David Moreau Simard <[email protected]>
Co-authored-by: Felix Fontein <[email protected]>

Author: dmsimard

.github/workflows/sanity-tests.yml

@@ -0,0 +1,48 @@
+name: Sanity tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  # Run once per week (Monday at 04:00 UTC)
+  schedule:
+    - cron: '0 4 * * 1'
+
+jobs:
+  build:
+    name: '${{ matrix.name }}'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: latest Ansible 5 release
+            ansible_major: 5
+            ansible_core: 2.12
+            ansible_core_next: 2.13
+          - name: latest Ansible 6 release
+            ansible_major: 6
+            ansible_core: 2.13
+            ansible_core_next: 2.14
+
+    steps:
+      - name: Check out ansible-build-data
+        uses: actions/checkout@v3
+        with:
+          path: ansible-build-data
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install ansible-core ${{ matrix.ansible_core }}
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install ansible-core>=${{ matrix.ansible_core }},${{ matrix.ansible_core_next }}
+
+      - name: "Run sanity tests for ${{ matrix.name }}"
+        run: |
+          ansible-playbook -vv tests/collection-tests.yaml -e 'galaxy_requirements="{{ playbook_dir | dirname }}/${{ matrix.ansible_major }}/galaxy-requirements.yaml"'
+        working-directory: ansible-build-data

tests/collection-tests.yaml

@@ -0,0 +1,49 @@
+---
+- name: Run tests on packaged collections
+  hosts: localhost
+  gather_facts: false
+  vars:
+    galaxy_requirements: "{{ playbook_dir | dirname }}/6/galaxy-requirements.yaml"
+    collections_path: /tmp/ansible_collections
+  tasks:
+    - name: Get list of installed packages to verify if podman is installed
+      package_facts:
+        manager: "auto"
+      # This is noisy (and potentially sensitive) to print on stdout
+      no_log: true
+
+    # This is so we can otherwise run unprivileged if podman is already installed
+    - when: "'podman' not in ansible_facts['packages'].keys()"
+      become: true
+      block:
+        - name: Install podman
+          package:
+            name: podman
+            state: present
+      rescue:
+        - name: Could not install podman
+          fail:
+            msg: |
+              Failed to elevate privileges and install podman.
+              Install podman manually or run ansible-playbook with elevated privileges.
+
+    - name: Install collections from galaxy
+      environment:
+        ANSIBLE_COLLECTIONS_PATH: "{{ collections_path }}"
+      command: ansible-galaxy collection install -r {{ galaxy_requirements }}
+      args:
+        creates: "{{ collections_path }}"
+
+    # ansible.builtin.find doesn't have mindepth
+    # https://github.com/ansible/ansible/issues/36369
+    - name: Find collection directories
+      command: find {{ collections_path }} -mindepth 2 -maxdepth 2 -type d
+      changed_when: false
+      register: _collection_directories
+
+    # Tests are broken up such that there is a task per collection (instead of one very long task)
+    - name: Run sanity tests
+      include_tasks: tasks/sanity-tests.yaml
+      loop: "{{ _collection_directories.stdout_lines }}"
+      loop_control:
+        loop_var: _collection_directory

tests/tasks/sanity-tests.yaml

@@ -0,0 +1,11 @@
+- name: "Run sanity tests for {{ _collection_name }}"
+  vars:
+    _collection_name: >-
+      {%- set _namespace = _collection_directory.split('/')[-2] -%}
+      {%- set _name = _collection_directory.split('/')[-1] -%}
+      {{ _namespace }}.{{ _name }}
+  command: ansible-test sanity --skip-test pylint --docker
+  changed_when: false
+  ignore_errors: true
+  args:
+    chdir: "{{ _collection_directory }}"