From c71fce1d876f079838d2da69006167e30b362055 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 17 May 2024 09:29:49 +0100
Subject: [PATCH 01/30] CI: Add Grafana and OpenSearch Dashboards variables for
 SOT

Depends-On: https://github.com/stackhpc/stackhpc-openstack-tests/pull/3
---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 200ac6711..5090c4c77 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -71,18 +71,30 @@
               -rfEx
               -vv
           environment:
+            GRAFANA_URL: "{{ sct_grafana_url }}"
+            GRAFANA_USERNAME: "{{ sct_grafana_username }}"
+            GRAFANA_PASSWORD: "{{ sct_grafana_password }}"
             OPENSEARCH_HOSTS: "{{ sct_opensearch_hosts }}"
             OPENSEARCH_PORT: "{{ sct_opensearch_port }}"
             OPENSEARCH_TLS: "{{ sct_opensearch_tls }}"
+            OPENSEARCH_DASHBOARDS_URL: "{{ sct_opensearch_dashboards_url }}"
+            OPENSEARCH_DASHBOARDS_USERNAME: "{{ sct_opensearch_dashboards_username }}"
+            OPENSEARCH_DASHBOARDS_PASSWORD: "{{ sct_opensearch_dashboards_password }}"
             PROMETHEUS_URL: "{{ sct_prometheus_url }}"
             PROMETHEUS_USERNAME: "{{ sct_prometheus_username }}"
             PROMETHEUS_PASSWORD: "{{ sct_prometheus_password }}"
           vars:
             kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
             kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}"
+            sct_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000"
+            sct_grafana_username: "grafana_local_admin"
+            sct_grafana_password: "{{ kolla_passwords.grafana_admin_password }}"
             sct_opensearch_hosts: "{{ kolla_internal_fqdn }}"
             sct_opensearch_port: 9200
             sct_opensearch_tls: false
+            sct_opensearch_dashboards_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:5601"
+            sct_opensearch_dashboards_username: "opensearch"
+            sct_opensearch_dashboards_password: "{{ kolla_passwords.opensearch_dashboards_password }}"
             sct_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
             sct_prometheus_username: admin
             sct_prometheus_password: "{{ kolla_passwords.prometheus_password }}"

From 6066bf5c216f95ae8601917379ad5b9574d56178 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 17 May 2024 09:31:21 +0100
Subject: [PATCH 02/30] CI: Add Docker and SELinux variables for SOT

Depends-On: https://github.com/stackhpc/stackhpc-openstack-tests/pull/4
---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 5090c4c77..2fe1184de 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -71,6 +71,8 @@
               -rfEx
               -vv
           environment:
+            DOCKER_VERSION_MIN: "{{ sct_docker_version_min }}"
+            DOCKER_VERSION_MAX: "{{ sct_docker_version_max }}"
             GRAFANA_URL: "{{ sct_grafana_url }}"
             GRAFANA_USERNAME: "{{ sct_grafana_username }}"
             GRAFANA_PASSWORD: "{{ sct_grafana_password }}"
@@ -83,9 +85,14 @@
             PROMETHEUS_URL: "{{ sct_prometheus_url }}"
             PROMETHEUS_USERNAME: "{{ sct_prometheus_username }}"
             PROMETHEUS_PASSWORD: "{{ sct_prometheus_password }}"
+            SELINUX_STATE: "{{ sct_selinux_state }}"
           vars:
             kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
             kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}"
+            # Inclusive min
+            sct_docker_version_min: "24.0.0"
+            # Exclusive max
+            sct_docker_version_max: "27.0.0"
             sct_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000"
             sct_grafana_username: "grafana_local_admin"
             sct_grafana_password: "{{ kolla_passwords.grafana_admin_password }}"
@@ -98,6 +105,7 @@
             sct_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
             sct_prometheus_username: admin
             sct_prometheus_password: "{{ kolla_passwords.prometheus_password }}"
+            sct_selinux_state: "{{ selinux_state }}"
       always:
         - name: Fetch results
           ansible.builtin.fetch:

From b92a9f29b859626721a4cb62a648d53267229ae6 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 17 May 2024 10:12:26 +0100
Subject: [PATCH 03/30] CI: Separate monitoring vs host test execution in SOT

There are currently two types of tests in StackHPC OpenStack Tests:
service-level tests such as those for monitoring, and host-level tests
such as those for Docker and SELinux. We should execute the
service-level tests from a single remote host, while the host-level
tests should be executed on each of the overcloud hosts.

This means that we now get several results files, so it becomes
important to provide a summary of failures.
---
 .github/workflows/stackhpc-all-in-one.yml     |  22 ++-
 etc/kayobe/ansible/stackhpc-cloud-tests.yml   |  81 +++++++--
 .../ansible/stackhpc-openstack-tests.yml      | 171 ++++++++++++++++++
 3 files changed, 250 insertions(+), 24 deletions(-)
 create mode 100644 etc/kayobe/ansible/stackhpc-openstack-tests.yml

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index f05b8a108..2f49ea1e0 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -496,16 +496,20 @@ jobs:
             sct-results/
         if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-cloud-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }}
 
-      - name: Fail if any Tempest tests failed
+      - name: Fail if any tests failed
         run: |
-          test $(wc -l < tempest-artifacts/failed-tests) -lt 1
-
-      - name: Fail if any StackHPC Cloud tests failed
-        run: |
-          echo "Some StackHPC Cloud tests failed."
-          echo "See HTML results artifact (sct-results) for details."
-          exit 1
-        if: steps.stackhpc-cloud-tests.outcome == 'failure'
+          rc=0
+          if [[ $(wc -l < tempest-artifacts/failed-tests) -ne 0 ]]; then
+            echo "Some Tempest tests failed."
+            echo "See HTML results artifact (tempest-artifacts) for details."
+            rc=1
+          fi
+          if [[ $(wc -l < sct-results/failed-tests) -ne 0 ]]; then
+            echo "Some StackHPC OpenStack tests failed."
+            echo "See HTML results artifact (sct-results) for details."
+            rc=1
+          fi
+          exit $rc
 
       - name: Destroy
         run: terraform destroy -auto-approve
diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 2fe1184de..f8309f15f 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -1,6 +1,6 @@
 ---
 - name: Run StackHPC Cloud tests
-  hosts: tempest_runner
+  hosts: tempest_runner:overcloud
   tags:
     - stackhpc-cloud-tests
   vars:
@@ -12,6 +12,11 @@
   tasks:
     - name: Stackhpc Cloud tests
       block:
+        - name: Assert that there is only one host in the tempest_runner group
+          assert:
+            that: groups.get('tempest_runner', []) | length == 1
+            fail_msg: The tempest_runner group should contain exactly one host
+
         - name: Create a temporary directory for tests repo
           ansible.builtin.tempfile:
             state: directory
@@ -60,19 +65,19 @@
             file: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
             name: kolla_passwords
 
-        - name: Run StackHPC Cloud tests
+        # Monitoring tests should run once, executed on the host in the
+        # tempest_runner group.
+        - name: Run StackHPC Cloud monitoring tests
           ansible.builtin.command:
             cmd: >
               {{ sct_venv }}/bin/py.test
-              --html={{ results_tmpdir.path }}/stackhpc-cloud-tests.html
+              --html={{ results_tmpdir.path }}/monitoring.html
               --self-contained-html
-              --pyargs stackhpc_cloud_tests
+              --pyargs stackhpc_cloud_tests.monitoring
               --timeout {{ sct_timeout }}
               -rfEx
               -vv
           environment:
-            DOCKER_VERSION_MIN: "{{ sct_docker_version_min }}"
-            DOCKER_VERSION_MAX: "{{ sct_docker_version_max }}"
             GRAFANA_URL: "{{ sct_grafana_url }}"
             GRAFANA_USERNAME: "{{ sct_grafana_username }}"
             GRAFANA_PASSWORD: "{{ sct_grafana_password }}"
@@ -85,14 +90,9 @@
             PROMETHEUS_URL: "{{ sct_prometheus_url }}"
             PROMETHEUS_USERNAME: "{{ sct_prometheus_username }}"
             PROMETHEUS_PASSWORD: "{{ sct_prometheus_password }}"
-            SELINUX_STATE: "{{ sct_selinux_state }}"
           vars:
             kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
             kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}"
-            # Inclusive min
-            sct_docker_version_min: "24.0.0"
-            # Exclusive max
-            sct_docker_version_max: "27.0.0"
             sct_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000"
             sct_grafana_username: "grafana_local_admin"
             sct_grafana_password: "{{ kolla_passwords.grafana_admin_password }}"
@@ -105,13 +105,64 @@
             sct_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
             sct_prometheus_username: admin
             sct_prometheus_password: "{{ kolla_passwords.prometheus_password }}"
+          failed_when: monitoring_results.rc not in [0, 1]
+          register: monitoring_results
+          when: "'tempest_runner' in group_names"
+
+        # Host tests should run on every host in the overcloud group.
+        # TODO: Use TestInfra's native Ansible or SSH connection plugins for
+        # remote test execution? That would place all results in a single file
+        # and allow us to execute all tests from a single host.
+        # https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends
+        - name: Run StackHPC Cloud host tests
+          ansible.builtin.command:
+            cmd: >
+              {{ sct_venv }}/bin/py.test
+              --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
+              --self-contained-html
+              --pyargs stackhpc_cloud_tests.host
+              --timeout {{ sct_timeout }}
+              -vv
+          environment:
+            DOCKER_VERSION_MIN: "{{ sct_docker_version_min }}"
+            DOCKER_VERSION_MAX: "{{ sct_docker_version_max }}"
+            SELINUX_STATE: "{{ sct_selinux_state }}"
+          vars:
+            # Inclusive min
+            sct_docker_version_min: "24.0.0"
+            # Exclusive max
+            sct_docker_version_max: "27.0.0"
             sct_selinux_state: "{{ selinux_state }}"
+          failed_when: host_results.rc not in [0, 1]
+          register: host_results
+          when: "'overcloud' in group_names"
       always:
-        - name: Fetch results
-          ansible.builtin.fetch:
-            src: "{{ results_tmpdir.path }}/stackhpc-cloud-tests.html"
+        - name: Synchronize results
+          ansible.posix.synchronize:
+            src: "{{ results_tmpdir.path }}/"
             dest: "{{ results_path_local }}/"
-            flat: true
+            mode: pull
+            archive: no
+            recursive: true
+            # For jump host
+            use_ssh_args: true
+        - name: Write a file containing failed test runs
+          ansible.builtin.copy:
+            content: |-
+              {% for host in ansible_play_hosts_all %}
+              {% if host not in ansible_play_hosts %}
+              {{ host }}: Host failure
+              {% endif %}
+              {% if hostvars[host].monitoring_results.rc | default(0) != 0 %}
+              monitoring.html
+              {% endif %}
+              {% if hostvars[host].host_results.rc | default(0) != 0 %}
+              host-{{ host }}.html
+              {% endif %}
+              {% endfor %}
+            dest: "{{ results_path_local }}/failed-tests"
+          delegate_to: localhost
+          run_once: true
 
         - name: Clean up temporary directory
           ansible.builtin.file:
diff --git a/etc/kayobe/ansible/stackhpc-openstack-tests.yml b/etc/kayobe/ansible/stackhpc-openstack-tests.yml
new file mode 100644
index 000000000..cb4ac6cd7
--- /dev/null
+++ b/etc/kayobe/ansible/stackhpc-openstack-tests.yml
@@ -0,0 +1,171 @@
+---
+# This playbook executes tests from the StackHPC OpenStack Tests repository.
+# https://github.com/stackhpc/stackhpc-openstack-tests
+
+- name: Run StackHPC OpenStack tests
+  hosts: tempest_runner:overcloud
+  tags:
+    - stackhpc-openstack-tests
+  vars:
+    sot_venv: "{{ virtualenv_path }}/sot-venv"
+    sot_repo: "https://github.com/stackhpc/stackhpc-openstack-tests"
+    sot_version: "v0.0.1"
+    sot_timeout: 30
+    results_path_local: "{{ lookup('env', 'HOME') }}/sot-results"
+  tasks:
+    - name: Assert that there is only one host in the tempest_runner group
+      assert:
+        that: groups.get('tempest_runner', []) | length == 1
+        fail_msg: The tempest_runner group should contain exactly one host
+
+    - block:
+        - name: Create a temporary directory for tests repo
+          ansible.builtin.tempfile:
+            state: directory
+            suffix: sot-repo
+          register: repo_tmpdir
+
+        - name: Create a temporary directory for results
+          ansible.builtin.tempfile:
+            state: directory
+            suffix: sot-results
+          register: results_tmpdir
+
+        - name: Clone the StackHPC OpenStack tests repository
+          ansible.builtin.git:
+            repo: "{{ sot_repo }}"
+            version: "{{ sot_version }}"
+            dest: "{{ repo_tmpdir.path }}"
+            depth: 1
+            single_branch: true
+
+        - name: Ensure the latest versions of pip and setuptools are installed # noqa package-latest
+          ansible.builtin.pip:
+            name: "{{ item.name }}"
+            state: latest
+            virtualenv: "{{ sot_venv }}"
+            virtualenv_command: "python3 -m venv"
+          with_items:
+            - { name: pip }
+            - { name: setuptools }
+
+        - name: Ensure required Python packages are installed
+          ansible.builtin.pip:
+            name:
+              - "{{ repo_tmpdir.path }}"
+              - "-r{{ repo_tmpdir.path }}/requirements.txt"
+              - pytest-html
+              - pytest-timeout
+            virtualenv: "{{ sot_venv }}"
+
+        - name: Include Kolla Ansible passwords
+          ansible.builtin.include_vars:
+            file: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
+            name: kolla_passwords
+
+        # Monitoring tests should run once, executed on the host in the
+        # tempest_runner group.
+        - name: Run StackHPC OpenStack monitoring tests
+          ansible.builtin.command:
+            cmd: >
+              {{ sot_venv }}/bin/py.test
+              --html={{ results_tmpdir.path }}/monitoring.html
+              --self-contained-html
+              --pyargs stackhpc_openstack_tests.monitoring
+              --timeout {{ sot_timeout }}
+              -vv
+          environment:
+            GRAFANA_URL: "{{ sot_grafana_url }}"
+            GRAFANA_USERNAME: "{{ sot_grafana_username }}"
+            GRAFANA_PASSWORD: "{{ sot_grafana_password }}"
+            OPENSEARCH_HOSTS: "{{ sot_opensearch_hosts }}"
+            OPENSEARCH_PORT: "{{ sot_opensearch_port }}"
+            OPENSEARCH_TLS: "{{ sot_opensearch_tls }}"
+            OPENSEARCH_DASHBOARDS_URL: "{{ sot_opensearch_dashboards_url }}"
+            OPENSEARCH_DASHBOARDS_USERNAME: "{{ sot_opensearch_dashboards_username }}"
+            OPENSEARCH_DASHBOARDS_PASSWORD: "{{ sot_opensearch_dashboards_password }}"
+            PROMETHEUS_URL: "{{ sot_prometheus_url }}"
+            PROMETHEUS_USERNAME: "{{ sot_prometheus_username }}"
+            PROMETHEUS_PASSWORD: "{{ sot_prometheus_password }}"
+          vars:
+            kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
+            kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}"
+            sot_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000"
+            sot_grafana_username: "grafana_local_admin"
+            sot_grafana_password: "{{ kolla_passwords.grafana_admin_password }}"
+            sot_opensearch_hosts: "{{ kolla_internal_fqdn }}"
+            sot_opensearch_port: 9200
+            sot_opensearch_tls: false
+            sot_opensearch_dashboards_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:5601"
+            sot_opensearch_dashboards_username: "opensearch"
+            sot_opensearch_dashboards_password: "{{ kolla_passwords.opensearch_dashboards_password }}"
+            sot_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
+            sot_prometheus_username: "admin"
+            sot_prometheus_password: "{{ kolla_passwords.prometheus_password }}"
+          failed_when: monitoring_results.rc not in [0, 1]
+          register: monitoring_results
+          when: "'tempest_runner' in group_names"
+
+        # Host tests should run on every host in the overcloud group.
+        # TODO: Use TestInfra's native Ansible or SSH connection plugins for
+        # remote test execution? That would place all results in a single file
+        # and allow us to execute all tests from a single host.
+        # https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends
+        - name: Run StackHPC OpenStack host tests
+          ansible.builtin.command:
+            cmd: >
+              {{ sot_venv }}/bin/py.test
+              --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
+              --self-contained-html
+              --pyargs stackhpc_openstack_tests.host
+              --timeout {{ sot_timeout }}
+              -vv
+          environment:
+            DOCKER_VERSION_MIN: "{{ sot_docker_version_min }}"
+            DOCKER_VERSION_MAX: "{{ sot_docker_version_max }}"
+            SELINUX_STATE: "{{ sot_selinux_state }}"
+          vars:
+            # Inclusive min
+            sot_docker_version_min: "24.0.0"
+            # Exclusive max
+            sot_docker_version_max: "27.0.0"
+            sot_selinux_state: "{{ selinux_state }}"
+          failed_when: host_results.rc not in [0, 1]
+          register: host_results
+          when: "'overcloud' in group_names"
+      always:
+        - name: Synchronize results
+          ansible.posix.synchronize:
+            src: "{{ results_tmpdir.path }}/"
+            dest: "{{ results_path_local }}/"
+            mode: pull
+            archive: no
+            recursive: true
+            # For jump host
+            use_ssh_args: true
+
+        - name: Write a file containing failed test runs
+          ansible.builtin.copy:
+            content: |-
+              {% for host in ansible_play_hosts_all %}
+              {% if host not in ansible_play_hosts %}
+              {{ host }}: Host failure
+              {% endif %}
+              {% if hostvars[host].monitoring_results.rc | default(0) != 0 %}
+              monitoring.html
+              {% endif %}
+              {% if hostvars[host].host_results.rc | default(0) != 0 %}
+              host-{{ host }}.html
+              {% endif %}
+              {% endfor %}
+            dest: "{{ results_path_local }}/failed-tests"
+          delegate_to: localhost
+          run_once: true
+
+        - name: Clean up temporary directory
+          ansible.builtin.file:
+            path: "{{ item }}"
+            state: absent
+          loop:
+            - "{{ repo_tmpdir.path }}"
+            - "{{ results_tmpdir.path }}"

From 4ed6d092b594c10c2372fa9ebc5cceced7b089da Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Fri, 17 May 2024 09:34:18 +0100
Subject: [PATCH 04/30] DNM: Slim CI, pin sot_version

---
 .github/workflows/stackhpc-pull-request.yml   | 91 +------------------
 .../ansible/stackhpc-openstack-tests.yml      |  3 +-
 2 files changed, 4 insertions(+), 90 deletions(-)

diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
index 65caaa0ce..5e90b0943 100644
--- a/.github/workflows/stackhpc-pull-request.yml
+++ b/.github/workflows/stackhpc-pull-request.yml
@@ -7,6 +7,8 @@ concurrency:
 name: Pull request
 'on':
   pull_request:
+  # FIXME: Remove
+  push:
 jobs:
   # Detect which files have changed and use this to run jobs conditionally.
   # Note that we can't use the workflow-level paths attribute since this
@@ -143,92 +145,3 @@ jobs:
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  all-in-one-ubuntu-jammy-ovn:
-    name: aio (Ubuntu Jammy OVN)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: ubuntu
-      os_release: jammy
-      ssh_username: ubuntu
-      neutron_plugin: ovn
-      OS_CLOUD: openstack
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-    secrets: inherit
-    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  all-in-one-rocky-9-ovs:
-    name: aio (Rocky 9 OVS)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: rocky
-      os_release: "9"
-      ssh_username: cloud-user
-      neutron_plugin: ovs
-      OS_CLOUD: openstack
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-    secrets: inherit
-    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  all-in-one-rocky-9-ovn:
-    name: aio (Rocky 9 OVN)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: rocky
-      os_release: "9"
-      ssh_username: cloud-user
-      neutron_plugin: ovn
-      OS_CLOUD: openstack
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-    secrets: inherit
-    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  # Test two upgrade scenarios: Ubuntu Jammy OVS and Rocky 9 OVN.
-
-  all-in-one-upgrade-ubuntu-jammy-ovs:
-    name: aio upgrade (Ubuntu Jammy OVS)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: ubuntu
-      os_release: jammy
-      ssh_username: ubuntu
-      neutron_plugin: ovs
-      OS_CLOUD: openstack
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-      upgrade: true
-    secrets: inherit
-    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
-
-  all-in-one-upgrade-rocky-9-ovn:
-    name: aio upgrade (Rocky 9 OVN)
-    needs:
-      - check-changes
-      - build-kayobe-image
-    uses: ./.github/workflows/stackhpc-all-in-one.yml
-    with:
-      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
-      os_distribution: rocky
-      os_release: "9"
-      ssh_username: cloud-user
-      neutron_plugin: ovn
-      OS_CLOUD: openstack
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
-      upgrade: true
-    secrets: inherit
-    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
diff --git a/etc/kayobe/ansible/stackhpc-openstack-tests.yml b/etc/kayobe/ansible/stackhpc-openstack-tests.yml
index cb4ac6cd7..8489cceac 100644
--- a/etc/kayobe/ansible/stackhpc-openstack-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-openstack-tests.yml
@@ -9,7 +9,8 @@
   vars:
     sot_venv: "{{ virtualenv_path }}/sot-venv"
     sot_repo: "https://github.com/stackhpc/stackhpc-openstack-tests"
-    sot_version: "v0.0.1"
+    # FIXME
+    sot_version: "docker-selinux"
     sot_timeout: 30
     results_path_local: "{{ lookup('env', 'HOME') }}/sot-results"
   tasks:

From 10fcf6f1670875e8620604f29d923a46cfc5fa79 Mon Sep 17 00:00:00 2001
From: Mark Goddard <mark@stackhpc.com>
Date: Mon, 20 May 2024 14:44:23 +0000
Subject: [PATCH 05/30] SOT: Support OpenSearch and Dashboards TLS

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml   |   4 +-
 .../ansible/stackhpc-openstack-tests.yml      | 172 ------------------
 2 files changed, 3 insertions(+), 173 deletions(-)
 delete mode 100644 etc/kayobe/ansible/stackhpc-openstack-tests.yml

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index f8309f15f..bc490ff6a 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -87,6 +87,7 @@
             OPENSEARCH_DASHBOARDS_URL: "{{ sct_opensearch_dashboards_url }}"
             OPENSEARCH_DASHBOARDS_USERNAME: "{{ sct_opensearch_dashboards_username }}"
             OPENSEARCH_DASHBOARDS_PASSWORD: "{{ sct_opensearch_dashboards_password }}"
+            OPENSEARCH_DASHBOARDS_CACERT: "{{ sct_opensearch_dashboards_cacert }}"
             PROMETHEUS_URL: "{{ sct_prometheus_url }}"
             PROMETHEUS_USERNAME: "{{ sct_prometheus_username }}"
             PROMETHEUS_PASSWORD: "{{ sct_prometheus_password }}"
@@ -98,10 +99,11 @@
             sct_grafana_password: "{{ kolla_passwords.grafana_admin_password }}"
             sct_opensearch_hosts: "{{ kolla_internal_fqdn }}"
             sct_opensearch_port: 9200
-            sct_opensearch_tls: false
+            sct_opensearch_tls: "{{ kolla_enable_tls_internal | bool }}"
             sct_opensearch_dashboards_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:5601"
             sct_opensearch_dashboards_username: "opensearch"
             sct_opensearch_dashboards_password: "{{ kolla_passwords.opensearch_dashboards_password }}"
+            sct_opensearch_dashboards_cacert: "TODO"
             sct_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
             sct_prometheus_username: admin
             sct_prometheus_password: "{{ kolla_passwords.prometheus_password }}"
diff --git a/etc/kayobe/ansible/stackhpc-openstack-tests.yml b/etc/kayobe/ansible/stackhpc-openstack-tests.yml
deleted file mode 100644
index 8489cceac..000000000
--- a/etc/kayobe/ansible/stackhpc-openstack-tests.yml
+++ /dev/null
@@ -1,172 +0,0 @@
----
-# This playbook executes tests from the StackHPC OpenStack Tests repository.
-# https://github.com/stackhpc/stackhpc-openstack-tests
-
-- name: Run StackHPC OpenStack tests
-  hosts: tempest_runner:overcloud
-  tags:
-    - stackhpc-openstack-tests
-  vars:
-    sot_venv: "{{ virtualenv_path }}/sot-venv"
-    sot_repo: "https://github.com/stackhpc/stackhpc-openstack-tests"
-    # FIXME
-    sot_version: "docker-selinux"
-    sot_timeout: 30
-    results_path_local: "{{ lookup('env', 'HOME') }}/sot-results"
-  tasks:
-    - name: Assert that there is only one host in the tempest_runner group
-      assert:
-        that: groups.get('tempest_runner', []) | length == 1
-        fail_msg: The tempest_runner group should contain exactly one host
-
-    - block:
-        - name: Create a temporary directory for tests repo
-          ansible.builtin.tempfile:
-            state: directory
-            suffix: sot-repo
-          register: repo_tmpdir
-
-        - name: Create a temporary directory for results
-          ansible.builtin.tempfile:
-            state: directory
-            suffix: sot-results
-          register: results_tmpdir
-
-        - name: Clone the StackHPC OpenStack tests repository
-          ansible.builtin.git:
-            repo: "{{ sot_repo }}"
-            version: "{{ sot_version }}"
-            dest: "{{ repo_tmpdir.path }}"
-            depth: 1
-            single_branch: true
-
-        - name: Ensure the latest versions of pip and setuptools are installed # noqa package-latest
-          ansible.builtin.pip:
-            name: "{{ item.name }}"
-            state: latest
-            virtualenv: "{{ sot_venv }}"
-            virtualenv_command: "python3 -m venv"
-          with_items:
-            - { name: pip }
-            - { name: setuptools }
-
-        - name: Ensure required Python packages are installed
-          ansible.builtin.pip:
-            name:
-              - "{{ repo_tmpdir.path }}"
-              - "-r{{ repo_tmpdir.path }}/requirements.txt"
-              - pytest-html
-              - pytest-timeout
-            virtualenv: "{{ sot_venv }}"
-
-        - name: Include Kolla Ansible passwords
-          ansible.builtin.include_vars:
-            file: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
-            name: kolla_passwords
-
-        # Monitoring tests should run once, executed on the host in the
-        # tempest_runner group.
-        - name: Run StackHPC OpenStack monitoring tests
-          ansible.builtin.command:
-            cmd: >
-              {{ sot_venv }}/bin/py.test
-              --html={{ results_tmpdir.path }}/monitoring.html
-              --self-contained-html
-              --pyargs stackhpc_openstack_tests.monitoring
-              --timeout {{ sot_timeout }}
-              -vv
-          environment:
-            GRAFANA_URL: "{{ sot_grafana_url }}"
-            GRAFANA_USERNAME: "{{ sot_grafana_username }}"
-            GRAFANA_PASSWORD: "{{ sot_grafana_password }}"
-            OPENSEARCH_HOSTS: "{{ sot_opensearch_hosts }}"
-            OPENSEARCH_PORT: "{{ sot_opensearch_port }}"
-            OPENSEARCH_TLS: "{{ sot_opensearch_tls }}"
-            OPENSEARCH_DASHBOARDS_URL: "{{ sot_opensearch_dashboards_url }}"
-            OPENSEARCH_DASHBOARDS_USERNAME: "{{ sot_opensearch_dashboards_username }}"
-            OPENSEARCH_DASHBOARDS_PASSWORD: "{{ sot_opensearch_dashboards_password }}"
-            PROMETHEUS_URL: "{{ sot_prometheus_url }}"
-            PROMETHEUS_USERNAME: "{{ sot_prometheus_username }}"
-            PROMETHEUS_PASSWORD: "{{ sot_prometheus_password }}"
-          vars:
-            kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
-            kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}"
-            sot_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000"
-            sot_grafana_username: "grafana_local_admin"
-            sot_grafana_password: "{{ kolla_passwords.grafana_admin_password }}"
-            sot_opensearch_hosts: "{{ kolla_internal_fqdn }}"
-            sot_opensearch_port: 9200
-            sot_opensearch_tls: false
-            sot_opensearch_dashboards_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:5601"
-            sot_opensearch_dashboards_username: "opensearch"
-            sot_opensearch_dashboards_password: "{{ kolla_passwords.opensearch_dashboards_password }}"
-            sot_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
-            sot_prometheus_username: "admin"
-            sot_prometheus_password: "{{ kolla_passwords.prometheus_password }}"
-          failed_when: monitoring_results.rc not in [0, 1]
-          register: monitoring_results
-          when: "'tempest_runner' in group_names"
-
-        # Host tests should run on every host in the overcloud group.
-        # TODO: Use TestInfra's native Ansible or SSH connection plugins for
-        # remote test execution? That would place all results in a single file
-        # and allow us to execute all tests from a single host.
-        # https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends
-        - name: Run StackHPC OpenStack host tests
-          ansible.builtin.command:
-            cmd: >
-              {{ sot_venv }}/bin/py.test
-              --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
-              --self-contained-html
-              --pyargs stackhpc_openstack_tests.host
-              --timeout {{ sot_timeout }}
-              -vv
-          environment:
-            DOCKER_VERSION_MIN: "{{ sot_docker_version_min }}"
-            DOCKER_VERSION_MAX: "{{ sot_docker_version_max }}"
-            SELINUX_STATE: "{{ sot_selinux_state }}"
-          vars:
-            # Inclusive min
-            sot_docker_version_min: "24.0.0"
-            # Exclusive max
-            sot_docker_version_max: "27.0.0"
-            sot_selinux_state: "{{ selinux_state }}"
-          failed_when: host_results.rc not in [0, 1]
-          register: host_results
-          when: "'overcloud' in group_names"
-      always:
-        - name: Synchronize results
-          ansible.posix.synchronize:
-            src: "{{ results_tmpdir.path }}/"
-            dest: "{{ results_path_local }}/"
-            mode: pull
-            archive: no
-            recursive: true
-            # For jump host
-            use_ssh_args: true
-
-        - name: Write a file containing failed test runs
-          ansible.builtin.copy:
-            content: |-
-              {% for host in ansible_play_hosts_all %}
-              {% if host not in ansible_play_hosts %}
-              {{ host }}: Host failure
-              {% endif %}
-              {% if hostvars[host].monitoring_results.rc | default(0) != 0 %}
-              monitoring.html
-              {% endif %}
-              {% if hostvars[host].host_results.rc | default(0) != 0 %}
-              host-{{ host }}.html
-              {% endif %}
-              {% endfor %}
-            dest: "{{ results_path_local }}/failed-tests"
-          delegate_to: localhost
-          run_once: true
-
-        - name: Clean up temporary directory
-          ansible.builtin.file:
-            path: "{{ item }}"
-            state: absent
-          loop:
-            - "{{ repo_tmpdir.path }}"
-            - "{{ results_tmpdir.path }}"

From dcdce514de15e14186f50ee21d2ae8f9049ffa47 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Fri, 31 Jan 2025 12:06:42 +0000
Subject: [PATCH 06/30] Minor fixes to SCT

---
 .github/workflows/stackhpc-all-in-one.yml   | 2 +-
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index 2f49ea1e0..b9476ab0a 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -468,7 +468,7 @@ jobs:
             -v $(pwd)/sct-results:/stack/sct-results \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
-            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-cloud-tests.yml' -e sot_version=${{ inputs.stackhpc_cloud_tests_version }}
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-cloud-tests.yml' -e sct_version=${{ inputs.stackhpc_cloud_tests_version }}
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index bc490ff6a..4528aeac3 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -6,7 +6,8 @@
   vars:
     sct_venv: "{{ virtualenv_path }}/sct-venv"
     sct_repo: https://github.com/stackhpc/stackhpc-cloud-tests
-    sct_version: main
+    # FIXME: Pin version for testing, remove before merge
+    sct_version: "docker-selinux"
     sct_timeout: 30
     results_path_local: "{{ lookup('env', 'HOME') }}/sct-results"
   tasks:

From e96f28f347920de4d84768fbd651e1522a6a4bc7 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Tue, 4 Feb 2025 10:20:36 +0000
Subject: [PATCH 07/30] Test SCT against the opensearch-dashboards-grafana
 branch

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 4528aeac3..7cb20201e 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -7,7 +7,8 @@
     sct_venv: "{{ virtualenv_path }}/sct-venv"
     sct_repo: https://github.com/stackhpc/stackhpc-cloud-tests
     # FIXME: Pin version for testing, remove before merge
-    sct_version: "docker-selinux"
+    #sct_version: "docker-selinux"
+    sct_version: "opensearch-dashboards-grafana"
     sct_timeout: 30
     results_path_local: "{{ lookup('env', 'HOME') }}/sct-results"
   tasks:

From e15448164a5f0e8140da413960b73075b73b0484 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Tue, 4 Feb 2025 15:33:11 +0000
Subject: [PATCH 08/30] Select test subset using expression and change the
 branch to pull in host tests on top of the monitoring tests

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 7cb20201e..b1b9edabb 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -7,8 +7,7 @@
     sct_venv: "{{ virtualenv_path }}/sct-venv"
     sct_repo: https://github.com/stackhpc/stackhpc-cloud-tests
     # FIXME: Pin version for testing, remove before merge
-    #sct_version: "docker-selinux"
-    sct_version: "opensearch-dashboards-grafana"
+    sct_version: "docker-selinux"
     sct_timeout: 30
     results_path_local: "{{ lookup('env', 'HOME') }}/sct-results"
   tasks:
@@ -75,7 +74,7 @@
               {{ sct_venv }}/bin/py.test
               --html={{ results_tmpdir.path }}/monitoring.html
               --self-contained-html
-              --pyargs stackhpc_cloud_tests.monitoring
+              -k monitoring
               --timeout {{ sct_timeout }}
               -rfEx
               -vv
@@ -124,7 +123,7 @@
               {{ sct_venv }}/bin/py.test
               --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
               --self-contained-html
-              --pyargs stackhpc_cloud_tests.host
+              -k host
               --timeout {{ sct_timeout }}
               -vv
           environment:

From 829f6bc8709c65838ffdc73783313ce87f18662d Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Tue, 4 Feb 2025 15:53:17 +0000
Subject: [PATCH 09/30] Comment to trigger SCT workflow on push

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index b1b9edabb..b3de5b1af 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -7,6 +7,7 @@
     sct_venv: "{{ virtualenv_path }}/sct-venv"
     sct_repo: https://github.com/stackhpc/stackhpc-cloud-tests
     # FIXME: Pin version for testing, remove before merge
+    # Comment - trigger SCT workflow on push
     sct_version: "docker-selinux"
     sct_timeout: 30
     results_path_local: "{{ lookup('env', 'HOME') }}/sct-results"

From 93af210181a50e767c01fe95b9eb4dfd5090414f Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Wed, 5 Feb 2025 12:20:19 +0000
Subject: [PATCH 10/30] In StackHPC cloud tests we're checking the host's
 SELinux state matches the targeted state  but we can't access what was
 defined upstream so we redefine 'selinux_state' in our group_vars - this must
 follow any upstream change.

---
 etc/kayobe/inventory/group_vars/all/selinux | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 etc/kayobe/inventory/group_vars/all/selinux

diff --git a/etc/kayobe/inventory/group_vars/all/selinux b/etc/kayobe/inventory/group_vars/all/selinux
new file mode 100644
index 000000000..80d084d7e
--- /dev/null
+++ b/etc/kayobe/inventory/group_vars/all/selinux
@@ -0,0 +1,5 @@
+---
+# Target SELinux state
+# NOTE(MaxN) In StackHPC cloud tests we're checking the host's SELinux state matches the targeted state
+# but we can't access what was defined upstream so we redefine here - this must follow any upstream change.
+selinux_state: permissive

From e3b21a219eb9e49c2bbd35dbd4acfd6048b832f2 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Wed, 5 Feb 2025 12:26:43 +0000
Subject: [PATCH 11/30] Restore the SCT pyargs else we don't find any tests to
 run.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index b3de5b1af..434f9c478 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -75,6 +75,7 @@
               {{ sct_venv }}/bin/py.test
               --html={{ results_tmpdir.path }}/monitoring.html
               --self-contained-html
+              --pyargs stackhpc_cloud_tests
               -k monitoring
               --timeout {{ sct_timeout }}
               -rfEx
@@ -124,6 +125,7 @@
               {{ sct_venv }}/bin/py.test
               --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
               --self-contained-html
+              --pyargs stackhpc_cloud_tests
               -k host
               --timeout {{ sct_timeout }}
               -vv

From 2c1d7d5c57292de8da50369253d2e1610ccdbd50 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Wed, 5 Feb 2025 15:36:31 +0000
Subject: [PATCH 12/30] Run all SCT tests - remove the monitoring and host
 filters to see the effect

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 434f9c478..2243d3070 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -76,10 +76,10 @@
               --html={{ results_tmpdir.path }}/monitoring.html
               --self-contained-html
               --pyargs stackhpc_cloud_tests
-              -k monitoring
               --timeout {{ sct_timeout }}
               -rfEx
               -vv
+          #    -k monitoring
           environment:
             GRAFANA_URL: "{{ sct_grafana_url }}"
             GRAFANA_USERNAME: "{{ sct_grafana_username }}"
@@ -126,9 +126,9 @@
               --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
               --self-contained-html
               --pyargs stackhpc_cloud_tests
-              -k host
               --timeout {{ sct_timeout }}
               -vv
+          #    -k monitoring
           environment:
             DOCKER_VERSION_MIN: "{{ sct_docker_version_min }}"
             DOCKER_VERSION_MAX: "{{ sct_docker_version_max }}"

From e9d849267ae7ec98d0b56c7db1ef5e5ffef481c2 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Mon, 10 Feb 2025 08:48:24 +0000
Subject: [PATCH 13/30] Instead of importing tests as modules into the venv we
 direct pytest to the subset (directory) of tests we want to run.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 2243d3070..7cbb52128 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -25,6 +25,8 @@
             suffix: sct-repo
           register: repo_tmpdir
 
+        - debug: var=repo_tmpdir
+
         - name: Create a temporary directory for results
           ansible.builtin.tempfile:
             state: directory
@@ -52,7 +54,6 @@
         - name: Ensure required individual Python packages are installed
           ansible.builtin.pip:
             name:
-              - "{{ repo_tmpdir.path }}"
               - pytest-html
               - pytest-timeout
             virtualenv: "{{ sct_venv }}"
@@ -67,6 +68,14 @@
             file: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
             name: kolla_passwords
 
+        - name: Print CWD
+          ansible.builtin.command:
+            cmd: >
+              /usr/bin/pwd
+          register: debug_working_dir
+
+        - debug: var=debug_working_dir
+
         # Monitoring tests should run once, executed on the host in the
         # tempest_runner group.
         - name: Run StackHPC Cloud monitoring tests
@@ -75,11 +84,10 @@
               {{ sct_venv }}/bin/py.test
               --html={{ results_tmpdir.path }}/monitoring.html
               --self-contained-html
-              --pyargs stackhpc_cloud_tests
               --timeout {{ sct_timeout }}
               -rfEx
               -vv
-          #    -k monitoring
+              "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/monitoring"
           environment:
             GRAFANA_URL: "{{ sct_grafana_url }}"
             GRAFANA_USERNAME: "{{ sct_grafana_username }}"
@@ -125,10 +133,9 @@
               {{ sct_venv }}/bin/py.test
               --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
               --self-contained-html
-              --pyargs stackhpc_cloud_tests
               --timeout {{ sct_timeout }}
               -vv
-          #    -k monitoring
+              "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/host"
           environment:
             DOCKER_VERSION_MIN: "{{ sct_docker_version_min }}"
             DOCKER_VERSION_MAX: "{{ sct_docker_version_max }}"

From 39f7968b86c7d0868a4bfab0a0dad9b4b5ba470f Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Mon, 10 Feb 2025 12:29:14 +0000
Subject: [PATCH 14/30] Print out the directory structure of the runner's SCT
 checkout to see why it can't find tests.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 7cbb52128..c168b750e 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -25,8 +25,6 @@
             suffix: sct-repo
           register: repo_tmpdir
 
-        - debug: var=repo_tmpdir
-
         - name: Create a temporary directory for results
           ansible.builtin.tempfile:
             state: directory
@@ -74,7 +72,22 @@
               /usr/bin/pwd
           register: debug_working_dir
 
-        - debug: var=debug_working_dir
+        # REMOVE!!
+        # Why does the runner fail to find the tests???
+        - name: Capture contents of the SCT repo
+          ansible.builtin.find:
+            path: "{{ repo_tmpdir.path }}"
+            file_type: "directory"
+            recurse: true
+            excludes: "{{ repo_tmpdir.path }}/.git"
+          register: contents
+        - name: Print the contents of the SCT repo
+          debug:
+            msg: "{{ item.path }}"
+          with_items:
+            - "{{ contents.files }}"
+          loop_control:
+            label: "{{ item.path }}"
 
         # Monitoring tests should run once, executed on the host in the
         # tempest_runner group.

From 5772f108d4521e698e92fcc82d7f8a0e0e7d5059 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Mon, 10 Feb 2025 13:30:42 +0000
Subject: [PATCH 15/30] Checkout the full SCT branch.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index c168b750e..fbc9b4fc3 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -36,8 +36,8 @@
             repo: "{{ sct_repo }}"
             version: "{{ sct_version }}"
             dest: "{{ repo_tmpdir.path }}"
-            depth: 1
-            single_branch: true
+            #depth: 1
+            #single_branch: true
 
         - name: Ensure the latest versions of pip and setuptools are installed # noqa package-latest
           ansible.builtin.pip:

From f42900039bc158b0226a2efe054600fa0d607db3 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Mon, 10 Feb 2025 14:59:47 +0000
Subject: [PATCH 16/30] Debugging why we're not checking out the tests on the
 runner.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index fbc9b4fc3..a147b71d7 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -8,7 +8,7 @@
     sct_repo: https://github.com/stackhpc/stackhpc-cloud-tests
     # FIXME: Pin version for testing, remove before merge
     # Comment - trigger SCT workflow on push
-    sct_version: "docker-selinux"
+    sct_version: "opensearch-dashboards-grafana"
     sct_timeout: 30
     results_path_local: "{{ lookup('env', 'HOME') }}/sct-results"
   tasks:
@@ -89,6 +89,15 @@
           loop_control:
             label: "{{ item.path }}"
 
+        # MORE DEBUG TO REMOVE
+        - name: get commit of of parent repository                                  
+          ansible.builtin.command:
+            cmd: git log
+            chdir: "{{ repo_tmpdir.path }}"
+          register: git_log                                    
+        - debug:
+            msg: "Log: {{git_log.stdout}}"
+
         # Monitoring tests should run once, executed on the host in the
         # tempest_runner group.
         - name: Run StackHPC Cloud monitoring tests

From 3bd23696faa2b526131591dcd028b4597b0275eb Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Mon, 10 Feb 2025 16:00:30 +0000
Subject: [PATCH 17/30] Debugging why we're not checking out the tests on the
 runner.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index a147b71d7..f0272447a 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -90,7 +90,16 @@
             label: "{{ item.path }}"
 
         # MORE DEBUG TO REMOVE
-        - name: get commit of of parent repository                                  
+        - name: Get status of SCT checkout
+          ansible.builtin.command:
+            cmd: git status
+            chdir: "{{ repo_tmpdir.path }}"
+          register: git_status
+        - debug:
+            msg: "Status: {{git_status.stdout}}"
+
+        # MORE DEBUG TO REMOVE
+        - name: Get log of SCT checkout
           ansible.builtin.command:
             cmd: git log
             chdir: "{{ repo_tmpdir.path }}"

From cbb96882d4f335cf1c4ff957cd0b1a8efe8fde57 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Mon, 10 Feb 2025 16:40:38 +0000
Subject: [PATCH 18/30] Override the runner's stackhpc_cloud_tests_version so
 we can test our SCT branch.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index f0272447a..ef4759ba7 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -6,9 +6,11 @@
   vars:
     sct_venv: "{{ virtualenv_path }}/sct-venv"
     sct_repo: https://github.com/stackhpc/stackhpc-cloud-tests
-    # FIXME: Pin version for testing, remove before merge
-    # Comment - trigger SCT workflow on push
-    sct_version: "opensearch-dashboards-grafana"
+    # Define the version of SCT used for testing, the github workflow overrides this with
+    # stackhpc_cloud_tests_version so this is only used if running "locally".
+    sct_version: "main"
+    # TODO: REMOVE THIS
+    sct_version_override: "docker-selinux"
     sct_timeout: 30
     results_path_local: "{{ lookup('env', 'HOME') }}/sct-results"
   tasks:
@@ -34,10 +36,10 @@
         - name: Clone the StackHPC Cloud tests repository
           ansible.builtin.git:
             repo: "{{ sct_repo }}"
-            version: "{{ sct_version }}"
+            version: "{{ sct_version_override }}"
             dest: "{{ repo_tmpdir.path }}"
-            #depth: 1
-            #single_branch: true
+            depth: 1
+            single_branch: true
 
         - name: Ensure the latest versions of pip and setuptools are installed # noqa package-latest
           ansible.builtin.pip:

From a32fba2bc0a4cdd25f3220c2d2cab972043c2cf7 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Mon, 10 Feb 2025 18:20:45 +0000
Subject: [PATCH 19/30] Remove debug and run the SCT host tests as root.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 45 ++++-----------------
 1 file changed, 7 insertions(+), 38 deletions(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index ef4759ba7..8443a0979 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -74,41 +74,6 @@
               /usr/bin/pwd
           register: debug_working_dir
 
-        # REMOVE!!
-        # Why does the runner fail to find the tests???
-        - name: Capture contents of the SCT repo
-          ansible.builtin.find:
-            path: "{{ repo_tmpdir.path }}"
-            file_type: "directory"
-            recurse: true
-            excludes: "{{ repo_tmpdir.path }}/.git"
-          register: contents
-        - name: Print the contents of the SCT repo
-          debug:
-            msg: "{{ item.path }}"
-          with_items:
-            - "{{ contents.files }}"
-          loop_control:
-            label: "{{ item.path }}"
-
-        # MORE DEBUG TO REMOVE
-        - name: Get status of SCT checkout
-          ansible.builtin.command:
-            cmd: git status
-            chdir: "{{ repo_tmpdir.path }}"
-          register: git_status
-        - debug:
-            msg: "Status: {{git_status.stdout}}"
-
-        # MORE DEBUG TO REMOVE
-        - name: Get log of SCT checkout
-          ansible.builtin.command:
-            cmd: git log
-            chdir: "{{ repo_tmpdir.path }}"
-          register: git_log                                    
-        - debug:
-            msg: "Log: {{git_log.stdout}}"
-
         # Monitoring tests should run once, executed on the host in the
         # tempest_runner group.
         - name: Run StackHPC Cloud monitoring tests
@@ -161,14 +126,15 @@
         # and allow us to execute all tests from a single host.
         # https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends
         - name: Run StackHPC Cloud host tests
-          ansible.builtin.command:
-            cmd: >
+          ansible.builtin.shell:
+            cmd: |
               {{ sct_venv }}/bin/py.test
               --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
               --self-contained-html
               --timeout {{ sct_timeout }}
               -vv
-              "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/host"
+              "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/host";
+              chmod 644 {{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
           environment:
             DOCKER_VERSION_MIN: "{{ sct_docker_version_min }}"
             DOCKER_VERSION_MAX: "{{ sct_docker_version_max }}"
@@ -182,6 +148,9 @@
           failed_when: host_results.rc not in [0, 1]
           register: host_results
           when: "'overcloud' in group_names"
+          # Some checks may need root hence we chmod the resulting html for reading
+          become: true
+
       always:
         - name: Synchronize results
           ansible.posix.synchronize:

From fdce8e33745cd3b45b636b9871d65781270e013b Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Mon, 10 Feb 2025 22:19:10 +0000
Subject: [PATCH 20/30] Make the SCT host test results readable - alternative
 way.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 8443a0979..355f690c8 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -126,15 +126,15 @@
         # and allow us to execute all tests from a single host.
         # https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends
         - name: Run StackHPC Cloud host tests
-          ansible.builtin.shell:
-            cmd: |
+          ansible.builtin.command:
+            cmd: >
               {{ sct_venv }}/bin/py.test
               --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
               --self-contained-html
               --timeout {{ sct_timeout }}
               -vv
-              "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/host";
-              chmod 644 {{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
+              "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/host"
+            #  chmod 644 {{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
           environment:
             DOCKER_VERSION_MIN: "{{ sct_docker_version_min }}"
             DOCKER_VERSION_MAX: "{{ sct_docker_version_max }}"
@@ -150,6 +150,11 @@
           when: "'overcloud' in group_names"
           # Some checks may need root hence we chmod the resulting html for reading
           become: true
+        # Host test results will be owned by root and unreadable - fix this
+        - name: Make StackHPC Cloud host test results readable
+          ansible.builtin.command:
+            cmd: chmod 644 {{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
+          become: true
 
       always:
         - name: Synchronize results

From 61817186760ab20bc078a5c1aa3f2ea6d1610366 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Tue, 11 Feb 2025 08:31:07 +0000
Subject: [PATCH 21/30] It's easier to change permissions on the host results
 than ownership.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 355f690c8..8f4086bee 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -148,12 +148,12 @@
           failed_when: host_results.rc not in [0, 1]
           register: host_results
           when: "'overcloud' in group_names"
-          # Some checks may need root hence we chmod the resulting html for reading
+          # Some host checks may need to run as root
           become: true
-        # Host test results will be owned by root and unreadable - fix this
-        - name: Make StackHPC Cloud host test results readable
+        # Host test results will be owned by root - we need to read and delete them
+        - name: Change permissions on SCT host test results
           ansible.builtin.command:
-            cmd: chmod 644 {{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
+            cmd: chmod 666 {{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
           become: true
 
       always:

From 0a588a163a8d85d2846aac793d396efe723de947 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Tue, 11 Feb 2025 08:31:58 +0000
Subject: [PATCH 22/30] Up the docker version range as our AIO docker is
 already newer than the docker test version.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 8f4086bee..39b0a3475 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -143,7 +143,7 @@
             # Inclusive min
             sct_docker_version_min: "24.0.0"
             # Exclusive max
-            sct_docker_version_max: "27.0.0"
+            sct_docker_version_max: "28.0.0"
             sct_selinux_state: "{{ selinux_state }}"
           failed_when: host_results.rc not in [0, 1]
           register: host_results

From 3d8b43dfc41979adebaa24d521b3ba14eb83a558 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Tue, 11 Feb 2025 09:39:13 +0000
Subject: [PATCH 23/30] Clean up temporary directories as root, host tests may
 have changed ownership to root.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 39b0a3475..a78282aa9 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -191,3 +191,5 @@
           loop:
             - "{{ repo_tmpdir.path }}"
             - "{{ results_tmpdir.path }}"
+          # Some files used by host tests may now be owned by root
+          become: true

From a1a3651326712eac8c8ad7dc0da9e6873a9472fc Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Tue, 11 Feb 2025 13:08:00 +0000
Subject: [PATCH 24/30] Only run host and/or monitoring tests if present, this
 allows for staged merging of different SCT branches.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index a78282aa9..2ca50bd72 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -10,7 +10,8 @@
     # stackhpc_cloud_tests_version so this is only used if running "locally".
     sct_version: "main"
     # TODO: REMOVE THIS
-    sct_version_override: "docker-selinux"
+    #sct_version_override: "docker-selinux"
+    sct_version_override: "prometheus-alerts"
     sct_timeout: 30
     results_path_local: "{{ lookup('env', 'HOME') }}/sct-results"
   tasks:
@@ -76,6 +77,10 @@
 
         # Monitoring tests should run once, executed on the host in the
         # tempest_runner group.
+        - name: Check for StackHPC Cloud monitoring tests
+          ansible.builtin.stat:
+            path: "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/monitoring"
+          register: stackhpc_cloud_monitoring_tests
         - name: Run StackHPC Cloud monitoring tests
           ansible.builtin.command:
             cmd: >
@@ -118,13 +123,17 @@
             sct_prometheus_password: "{{ kolla_passwords.prometheus_password }}"
           failed_when: monitoring_results.rc not in [0, 1]
           register: monitoring_results
-          when: "'tempest_runner' in group_names"
+          when: "'tempest_runner' in group_names and stackhpc_cloud_monitoring_tests.stat.exists"
 
         # Host tests should run on every host in the overcloud group.
         # TODO: Use TestInfra's native Ansible or SSH connection plugins for
         # remote test execution? That would place all results in a single file
         # and allow us to execute all tests from a single host.
         # https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends
+        - name: Check for StackHPC Cloud monitoring tests
+          ansible.builtin.stat:
+            path: "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/host"
+          register: stackhpc_cloud_host_tests
         - name: Run StackHPC Cloud host tests
           ansible.builtin.command:
             cmd: >
@@ -147,14 +156,15 @@
             sct_selinux_state: "{{ selinux_state }}"
           failed_when: host_results.rc not in [0, 1]
           register: host_results
-          when: "'overcloud' in group_names"
           # Some host checks may need to run as root
           become: true
+          when: "'overcloud' in group_names and stackhpc_cloud_host_tests.stat.exists"
         # Host test results will be owned by root - we need to read and delete them
         - name: Change permissions on SCT host test results
           ansible.builtin.command:
             cmd: chmod 666 {{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
           become: true
+          when: "'overcloud' in group_names and stackhpc_cloud_host_tests.stat.exists"
 
       always:
         - name: Synchronize results

From 2cd12a994a1d405c56c98e797217bc1945460e3f Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Tue, 11 Feb 2025 15:37:14 +0000
Subject: [PATCH 25/30] Tidy up and check against stackhpc/2024.1 to ensure we
 don't erroneously remove anything.

---
 .github/workflows/stackhpc-all-in-one.yml   |  2 +-
 .github/workflows/stackhpc-pull-request.yml | 89 +++++++++++++++++++++
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 13 +--
 3 files changed, 92 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
index b9476ab0a..3b01181bf 100644
--- a/.github/workflows/stackhpc-all-in-one.yml
+++ b/.github/workflows/stackhpc-all-in-one.yml
@@ -505,7 +505,7 @@ jobs:
             rc=1
           fi
           if [[ $(wc -l < sct-results/failed-tests) -ne 0 ]]; then
-            echo "Some StackHPC OpenStack tests failed."
+            echo "Some StackHPC Cloud tests failed."
             echo "See HTML results artifact (sct-results) for details."
             rc=1
           fi
diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
index 5e90b0943..d640e50a6 100644
--- a/.github/workflows/stackhpc-pull-request.yml
+++ b/.github/workflows/stackhpc-pull-request.yml
@@ -145,3 +145,92 @@ jobs:
       if: ${{ needs.check-changes.outputs.aio == 'true' }}
     secrets: inherit
     if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+
+  all-in-one-ubuntu-jammy-ovn:
+    name: aio (Ubuntu Jammy OVN)
+    needs:
+      - check-changes
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: ubuntu
+      os_release: jammy
+      ssh_username: ubuntu
+      neutron_plugin: ovn
+      OS_CLOUD: openstack
+      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+    secrets: inherit
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+
+  all-in-one-rocky-9-ovs:
+    name: aio (Rocky 9 OVS)
+    needs:
+      - check-changes
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: rocky
+      os_release: "9"
+      ssh_username: cloud-user
+      neutron_plugin: ovs
+      OS_CLOUD: openstack
+      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+    secrets: inherit
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+
+  all-in-one-rocky-9-ovn:
+    name: aio (Rocky 9 OVN)
+    needs:
+      - check-changes
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: rocky
+      os_release: "9"
+      ssh_username: cloud-user
+      neutron_plugin: ovn
+      OS_CLOUD: openstack
+      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+    secrets: inherit
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+
+  # Test two upgrade scenarios: Ubuntu Jammy OVS and Rocky 9 OVN.
+
+  all-in-one-upgrade-ubuntu-jammy-ovs:
+    name: aio upgrade (Ubuntu Jammy OVS)
+    needs:
+      - check-changes
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: ubuntu
+      os_release: jammy
+      ssh_username: ubuntu
+      neutron_plugin: ovs
+      OS_CLOUD: openstack
+      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+      upgrade: true
+    secrets: inherit
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+
+  all-in-one-upgrade-rocky-9-ovn:
+    name: aio upgrade (Rocky 9 OVN)
+    needs:
+      - check-changes
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: rocky
+      os_release: "9"
+      ssh_username: cloud-user
+      neutron_plugin: ovn
+      OS_CLOUD: openstack
+      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+      upgrade: true
+    secrets: inherit
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 2ca50bd72..dfdf72ca7 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -8,10 +8,7 @@
     sct_repo: https://github.com/stackhpc/stackhpc-cloud-tests
     # Define the version of SCT used for testing, the github workflow overrides this with
     # stackhpc_cloud_tests_version so this is only used if running "locally".
-    sct_version: "main"
-    # TODO: REMOVE THIS
-    #sct_version_override: "docker-selinux"
-    sct_version_override: "prometheus-alerts"
+    sct_version: main
     sct_timeout: 30
     results_path_local: "{{ lookup('env', 'HOME') }}/sct-results"
   tasks:
@@ -37,7 +34,7 @@
         - name: Clone the StackHPC Cloud tests repository
           ansible.builtin.git:
             repo: "{{ sct_repo }}"
-            version: "{{ sct_version_override }}"
+            version: "{{ sct_version }}"
             dest: "{{ repo_tmpdir.path }}"
             depth: 1
             single_branch: true
@@ -69,12 +66,6 @@
             file: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
             name: kolla_passwords
 
-        - name: Print CWD
-          ansible.builtin.command:
-            cmd: >
-              /usr/bin/pwd
-          register: debug_working_dir
-
         # Monitoring tests should run once, executed on the host in the
         # tempest_runner group.
         - name: Check for StackHPC Cloud monitoring tests

From dc7bef6aec7a6931bea08cbaf46968d7aa8c9cf8 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Wed, 12 Feb 2025 15:51:33 +0000
Subject: [PATCH 26/30] Remove workflow tweak for triggering AIO testing upon
 push.

---
 .github/workflows/stackhpc-pull-request.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
index d640e50a6..65caaa0ce 100644
--- a/.github/workflows/stackhpc-pull-request.yml
+++ b/.github/workflows/stackhpc-pull-request.yml
@@ -7,8 +7,6 @@ concurrency:
 name: Pull request
 'on':
   pull_request:
-  # FIXME: Remove
-  push:
 jobs:
   # Detect which files have changed and use this to run jobs conditionally.
   # Note that we can't use the workflow-level paths attribute since this

From a7076b3a78a192d7a799824c60311720bf689909 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Thu, 13 Feb 2025 09:55:51 +0000
Subject: [PATCH 27/30] Use FQCN of ansible.builtin.assert in Tempest runner
 host check.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index dfdf72ca7..4c9c2b627 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -15,7 +15,7 @@
     - name: Stackhpc Cloud tests
       block:
         - name: Assert that there is only one host in the tempest_runner group
-          assert:
+          ansible.builtin.assert:
             that: groups.get('tempest_runner', []) | length == 1
             fail_msg: The tempest_runner group should contain exactly one host
 

From 100ceb5bfd8b06c46f2be716c77db2fdd96607b3 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Mon, 17 Feb 2025 14:01:48 +0000
Subject: [PATCH 28/30] Tidy up by separating all ansible tasks with a new line
 and remove a commented out line.

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 4c9c2b627..405c6ee34 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -72,6 +72,7 @@
           ansible.builtin.stat:
             path: "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/monitoring"
           register: stackhpc_cloud_monitoring_tests
+
         - name: Run StackHPC Cloud monitoring tests
           ansible.builtin.command:
             cmd: >
@@ -121,10 +122,11 @@
         # remote test execution? That would place all results in a single file
         # and allow us to execute all tests from a single host.
         # https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends
-        - name: Check for StackHPC Cloud monitoring tests
+        - name: Check for StackHPC Cloud host tests
           ansible.builtin.stat:
             path: "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/host"
           register: stackhpc_cloud_host_tests
+
         - name: Run StackHPC Cloud host tests
           ansible.builtin.command:
             cmd: >
@@ -134,7 +136,6 @@
               --timeout {{ sct_timeout }}
               -vv
               "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/host"
-            #  chmod 644 {{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
           environment:
             DOCKER_VERSION_MIN: "{{ sct_docker_version_min }}"
             DOCKER_VERSION_MAX: "{{ sct_docker_version_max }}"
@@ -150,6 +151,7 @@
           # Some host checks may need to run as root
           become: true
           when: "'overcloud' in group_names and stackhpc_cloud_host_tests.stat.exists"
+
         # Host test results will be owned by root - we need to read and delete them
         - name: Change permissions on SCT host test results
           ansible.builtin.command:
@@ -167,6 +169,7 @@
             recursive: true
             # For jump host
             use_ssh_args: true
+
         - name: Write a file containing failed test runs
           ansible.builtin.copy:
             content: |-

From dd973a2e9241991dbc27333754e6868b73cd3462 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Mon, 17 Feb 2025 14:03:05 +0000
Subject: [PATCH 29/30] Remove sct_opensearch_dashboards_cacert: "TODO",
 there's nothing in StackHPC cloud tests to indicate its intention and if
 we're adding this as a placeholder then why not do the same for Prometheus
 and Grafana?

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 405c6ee34..18e731414 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -109,7 +109,6 @@
             sct_opensearch_dashboards_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:5601"
             sct_opensearch_dashboards_username: "opensearch"
             sct_opensearch_dashboards_password: "{{ kolla_passwords.opensearch_dashboards_password }}"
-            sct_opensearch_dashboards_cacert: "TODO"
             sct_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
             sct_prometheus_username: admin
             sct_prometheus_password: "{{ kolla_passwords.prometheus_password }}"

From 7508c766a950885f89e5d0e6aa4ff313282b4a93 Mon Sep 17 00:00:00 2001
From: Max Norton <maxn@stackhpc.com>
Date: Tue, 18 Feb 2025 11:29:55 +0000
Subject: [PATCH 30/30] Having removed sct_opensearch_dashboards_cacert we also
 need to remove OPENSEARCH_DASHBOARDS_CACERT

---
 etc/kayobe/ansible/stackhpc-cloud-tests.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
index 18e731414..ee0d513d1 100644
--- a/etc/kayobe/ansible/stackhpc-cloud-tests.yml
+++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -93,7 +93,6 @@
             OPENSEARCH_DASHBOARDS_URL: "{{ sct_opensearch_dashboards_url }}"
             OPENSEARCH_DASHBOARDS_USERNAME: "{{ sct_opensearch_dashboards_username }}"
             OPENSEARCH_DASHBOARDS_PASSWORD: "{{ sct_opensearch_dashboards_password }}"
-            OPENSEARCH_DASHBOARDS_CACERT: "{{ sct_opensearch_dashboards_cacert }}"
             PROMETHEUS_URL: "{{ sct_prometheus_url }}"
             PROMETHEUS_USERNAME: "{{ sct_prometheus_username }}"
             PROMETHEUS_PASSWORD: "{{ sct_prometheus_password }}"