Document applying integration tests to deployments (azimuth-cloud#129)

* Initial commit of testing docs

* First pass at testing docs

* Add run-tests script

* Add test jobs to the sample GitLab config + document

* Add testing to best practice doc

* Add documentation about generating an unrestricted appcred

* Add small clarification

* Remove test generation from run-tests script

* Rework CI jobs to better support multiple appcreds for testing

* Incorporate review suggestions

Author: Matt Pryor

.github/environments/arcus-ha/ansible.cfg

@@ -1,5 +1,5 @@
 [defaults]
-inventory = ../../../environments/base/inventory,../../../environments/ha/inventory,../../../environments/demo/inventory,../arcus/inventory,./inventory
+inventory = ../../../environments/base/inventory,../../../environments/ha/inventory,../../../environments/demo/inventory,../common/inventory,../arcus/inventory,./inventory
 roles_path = ../../../.ansible/roles
 collections_path = ../../../.ansible/collections
 

.github/environments/arcus/ansible.cfg

@@ -1,5 +1,5 @@
 [defaults]
-inventory = ../../../environments/base/inventory,../../../environments/singlenode/inventory,../../../environments/demo/inventory,./inventory
+inventory = ../../../environments/base/inventory,../../../environments/singlenode/inventory,../../../environments/demo/inventory,../common/inventory,./inventory
 roles_path = ../../../.ansible/roles
 collections_path = ../../../.ansible/collections
 

.github/environments/common/inventory/group_vars/all.yml

@@ -0,0 +1,33 @@
+---
+
+#####
+# This file defines the test cases used in CI
+#####
+
+#-----
+# Settings for the CaaS test suite
+#-----
+# Only generate tests for specifically enabled cluster types
+generate_tests_caas_default_test_case_enabled: false
+
+# Enable the workstation, slurm, repo2docker and rstudio test cases
+generate_tests_caas_test_case_workstation_enabled: true
+generate_tests_caas_test_case_slurm_enabled: true
+generate_tests_caas_test_case_repo2docker_enabled: true
+generate_tests_caas_test_case_rstudio_enabled: true
+
+#-----
+# Settings for the Kubernetes test suite
+#-----
+# Generate test cases for the most recent Kubernetes versions only
+generate_tests_kubernetes_test_cases_latest_only: true
+
+#-----
+# Settings for the Kubernetes apps test suite
+#-----
+# Only generate tests for specifically enabled app templates
+generate_tests_kubernetes_apps_default_test_case_enabled: false
+
+# Enable the jupyterhub and daskhub apps
+generate_tests_kubernetes_apps_test_case_jupyterhub_enabled: true
+generate_tests_kubernetes_apps_test_case_daskhub_enabled: true

.github/environments/common/inventory/hosts

@@ -0,0 +1,7 @@
+# This file exists so that Ansible treats the directory as an inventory
+
+[terraform_provision]
+# This group should contain the hosts that are used to provision the infrastructure
+# onto which Azimuth will be deployed
+# E.g.:
+# localhost  ansible_connection=local  ansible_python_interpreter="{{ ansible_playbook_python }}"

.github/environments/leafcloud-ha/ansible.cfg

@@ -1,5 +1,5 @@
 [defaults]
-inventory = ../../../environments/base/inventory,../../../environments/ha/inventory,../../../environments/demo/inventory,../leafcloud/inventory,./inventory
+inventory = ../../../environments/base/inventory,../../../environments/ha/inventory,../../../environments/demo/inventory,../common/inventory,../leafcloud/inventory,./inventory
 roles_path = ../../../.ansible/roles
 collections_path = ../../../.ansible/collections
 

.github/environments/leafcloud/ansible.cfg

@@ -1,5 +1,5 @@
 [defaults]
-inventory = ../../../environments/base/inventory,../../../environments/singlenode/inventory,../../../environments/demo/inventory,./inventory
+inventory = ../../../environments/base/inventory,../../../environments/singlenode/inventory,../../../environments/demo/inventory,../common/inventory,./inventory
 roles_path = ../../../.ansible/roles
 collections_path = ../../../.ansible/collections
 

.gitlab-ci.yml.sample

@@ -7,8 +7,6 @@ variables:
   # Because we are installing git-crypt as part of the job, we cannot reuse old
   # checkouts where git-crypt is already initialised as this results in an error
   GIT_STRATEGY: clone
-  # Write the Azimuth URL to an envfile in the local directory
-  AZIMUTH_URL_ENVFILE: ${CI_PROJECT_DIR}/azimuth.env
   # Use the pipeline credentials for Terraform
   # This assumes that we are using GitLab-managed Terraform state (recommended when available)
   TF_HTTP_USERNAME: gitlab-ci-token
@@ -19,10 +17,16 @@ stages:
   - scheduled
   # This stage owns the deploy and teardown jobs for dynamic environments
   - aio
-  # This stage owns the deploy job for our staging environment
+  # This stage runs tests for dynamic environments
+  - aio_test
+  # This stage owns the deploy job for the staging environment
   - staging
-  # This stage owns the deploy job for our production environment
+  # This stage owns the test job for the staging environment
+  - staging_test
+  # This stage owns the deploy job for the production environment
   - production
+  # This stage owns the test job for the production environment
+  - production_test
 
 #####
 # This job checks to see if there is a new release that needs to be merged
@@ -76,9 +80,7 @@ check_for_release:
         fi
 
 #####
-# This job deploys a dynamic review environment for the current branch
-#
-# Runs for every commit to every branch except main
+# This job deploys a dynamic review environment for a merge request
 #
 # Each environment is a single node deployment that is isolated from the other
 # branches with it's own GitLab environment, Terraform state and OpenStack resources
@@ -98,8 +100,8 @@ deploy_aio:
       when: never
     # Allow deployments to be manually triggered even when there are no changed files
     - if: $CI_COMMIT_BRANCH && $CI_PIPELINE_SOURCE == "web"
-    # Run when there is a push to a branch that is not main changes one of the relevant files
-    - if: $CI_COMMIT_BRANCH && $CI_PIPELINE_SOURCE == "push"
+    # Run when there is a push to a branch with a merge request that changes one of the relevant files
+    - if: $CI_COMMIT_BRANCH && $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
         # Files that affect the aio environment
         - env
@@ -109,19 +111,52 @@ deploy_aio:
         - environments/singlenode/**/*
         - environments/site/**/*
         - environments/aio/**/*
-  artifacts:
-    reports:
-      dotenv: azimuth.env
   environment:
     name: aio/$CI_COMMIT_REF_SLUG
-    url: $AZIMUTH_URL
     on_stop: stop_aio
   variables:
     # This is normally taken from the GitLab environment name, but in this case that
     # depends on the branch name so we need to be explicit about the config to use
     AZIMUTH_CONFIG_ENVIRONMENT: aio
+    ANSIBLE_FORCE_COLOR: true
+  before_script:
+    - source ./bin/ci-setup
   script:
-    - ./bin/ci-exec provision
+    - ansible-playbook stackhpc.azimuth_ops.provision
+
+#####
+# This job executes tests against dynamic review environments each time they are deployed
+#####
+test_aio:
+  stage: aio_test
+  rules:
+    # Do not run for commits to main
+    - if: $CI_COMMIT_BRANCH == "main"
+      when: never
+    # Allow deployments to be manually triggered even when there are no changed files
+    - if: $CI_COMMIT_BRANCH && $CI_PIPELINE_SOURCE == "web"
+    # Run when there is a push to a branch with a merge request that changes one of the relevant files
+    - if: $CI_COMMIT_BRANCH && $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        # Files that affect the aio environment
+        - env
+        - env.secret
+        - requirements.yml
+        - environments/base/**/*
+        - environments/singlenode/**/*
+        - environments/site/**/*
+        - environments/aio/**/*
+  environment:
+    name: aio/$CI_COMMIT_REF_SLUG
+  variables:
+    AZIMUTH_CONFIG_ENVIRONMENT: aio
+    ANSIBLE_FORCE_COLOR: true
+  before_script:
+    - source ./bin/ci-setup
+  script:
+    - ansible-playbook stackhpc.azimuth_ops.generate_tests
+    # Use the unrestricted application credential to execute the tests, as per the docs
+    - OS_CLOUD=unrestricted ./bin/run-tests
 
 #####
 # This job tears down dynamic review environments
@@ -139,7 +174,7 @@ stop_aio:
     - if: $CI_COMMIT_BRANCH && $CI_PIPELINE_SOURCE == "web"
       when: manual
       allow_failure: true
-    - if: $CI_COMMIT_BRANCH && $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_COMMIT_BRANCH && $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
         # Files that affect the aio environment
         - env
@@ -157,14 +192,16 @@ stop_aio:
   variables:
     AZIMUTH_CONFIG_ENVIRONMENT: aio
     GIT_STRATEGY: none
+    ANSIBLE_FORCE_COLOR: true
   before_script:
     - git clone ${CI_REPOSITORY_URL} ${CI_PROJECT_NAME}
     - cd ${CI_PROJECT_NAME}
     # Use the commit SHA if it still exists because git garbage collection hasn't run yet
     # If not, assume the branch has been merged and run the destroy from main instead
     - git checkout ${CI_COMMIT_SHA} || git checkout main
+    - source ./bin/ci-setup
   script:
-    - ./bin/ci-exec destroy
+    - ansible-playbook stackhpc.azimuth_ops.destroy
 
 #####
 # This job deploys Azimuth to the staging environment
@@ -192,14 +229,48 @@ deploy_staging:
         - environments/site/**/*
         - environments/site-ha/**/*
         - environments/staging/**/*
-  artifacts:
-    reports:
-      dotenv: azimuth.env
   environment:
     name: staging
-    url: $AZIMUTH_URL
+  variables:
+    ANSIBLE_FORCE_COLOR: true
+  before_script:
+    - source ./bin/ci-setup
+  script:
+    - ansible-playbook stackhpc.azimuth_ops.provision
+
+#####
+# This job executes tests against the staging environment each time it is deployed
+#####
+test_staging:
+  stage: staging_test
+  rules:
+    # Prevent the job from running on any branch that is not main
+    - if: $CI_COMMIT_BRANCH != "main"
+      when: never
+    # Allow deployments to be manually triggered on main even when there are no changed files
+    - if: $CI_PIPELINE_SOURCE == "web"
+    # Run for commits to main that change particular files
+    - if: $CI_PIPELINE_SOURCE == "push"
+      changes:
+        # Files that affect the staging environment
+        - env
+        - env.secret
+        - requirements.yml
+        - environments/base/**/*
+        - environments/ha/**/*
+        - environments/site/**/*
+        - environments/site-ha/**/*
+        - environments/staging/**/*
+  environment:
+    name: staging
+  variables:
+    ANSIBLE_FORCE_COLOR: true
+  before_script:
+    - source ./bin/ci-setup
   script:
-    - ./bin/ci-exec provision
+    - ansible-playbook stackhpc.azimuth_ops.generate_tests
+    # Use the unrestricted application credential to execute the tests, as per the docs
+    - OS_CLOUD=unrestricted ./bin/run-tests
 
 #####
 # This job deploys Azimuth to the production environment
@@ -231,11 +302,41 @@ deploy_production:
         - environments/site-ha/**/*
         - environments/production/**/*
       when: manual
-  artifacts:
-    reports:
-      dotenv: azimuth.env
   environment:
     name: production
-    url: $AZIMUTH_URL
+  variables:
+    ANSIBLE_FORCE_COLOR: true
+  before_script:
+    - source ./bin/ci-setup
+  script:
+    - ansible-playbook stackhpc.azimuth_ops.provision
+
+#####
+# This job executes tests against the production environment each time it is deployed
+#####
+test_production:
+  stage: production_test
+  rules:
+    - if: $CI_COMMIT_BRANCH != "main"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - env
+        - env.secret
+        - requirements.yml
+        - environments/base/**/*
+        - environments/ha/**/*
+        - environments/site/**/*
+        - environments/site-ha/**/*
+        - environments/production/**/*
+  environment:
+    name: production
+  variables:
+    ANSIBLE_FORCE_COLOR: true
+  before_script:
+    - source ./bin/ci-setup
   script:
-    - ./bin/ci-exec provision
+    - ansible-playbook stackhpc.azimuth_ops.generate_tests
+    # Use the unrestricted application credential to execute the tests, as per the docs
+    - OS_CLOUD=unrestricted ./bin/run-tests

bin/ci-exec renamed to bin/ci-setup

@@ -1,25 +1,11 @@
 #!/usr/bin/env bash
 
 #####
-# This script is designed to be run in a CI pipeline in an isolated Ubuntu Jammy environment
+# This script performs setup for a CI job in a clean Ubuntu Jammy environment
 #####
 
 set -eo pipefail
 
-case "$1" in
-  provision | destroy)
-    echo "Executing operation '$1'..."
-    AZIMUTH_OPERATION="$1"
-    ;;
-  *)
-    echo "No operation given - please specify either 'provision' or 'destroy'"
-    exit 1
-    ;;
-esac
-
-# Now we have processed the operation, drop the argument
-shift
-
 # Runs apt with sudo only if required
 # Required for environments where sudo is not present
 function run_apt {
@@ -55,11 +41,9 @@ fi
 
 run_apt update
 run_apt install -y git python3 python3-pip qemu-utils
+
 pip install -U pip
 pip install -r requirements.txt
 
 source ./bin/activate "$AZIMUTH_CONFIG_ENVIRONMENT" "$AZIMUTH_ENVIRONMENT"
 ansible-galaxy install -f -r requirements.yml
-# Force Ansible to print colours for nicer logs
-export ANSIBLE_FORCE_COLOR=true
-exec ansible-playbook "stackhpc.azimuth_ops.${AZIMUTH_OPERATION}" "$@"

bin/run-tests

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+#####
+## This script generates and then runs tests for the active environment
+#####
+
+set -exo pipefail
+
+
+if [ -z "$AZIMUTH_CONFIG_ROOT" ] || [ -z "$AZIMUTH_CONFIG_ENVIRONMENT_ROOT" ]; then
+    echo "Please activate an environment" >&2
+    exit 1
+fi
+
+
+ansible_variable() {
+    ANSIBLE_LOAD_CALLBACK_PLUGINS=true \
+    ANSIBLE_STDOUT_CALLBACK=json \
+    ansible -m debug -a "var=$1" all | \
+      jq -r ".plays[0].tasks[0].hosts.localhost.$1"
+}
+
+
+# Get the test directory
+test_directory="$(ansible_variable generate_tests_suite_directory)"
+# Use a headless browser by default
+export MOZ_HEADLESS="${MOZ_HEADLESS:-"1"}"
+# Execute the test suite, including any args we were given
+exec robot --loglevel debug --consolecolors on "$@" "$test_directory"

docs/best-practice.md

@@ -110,6 +110,11 @@ Merging to `main` will also create a job to deploy to `production`, but that job
 manual trigger. Once the change has been validated in `staging`, the job to deploy to `production`
 can be actioned.
 
+It is also possible to run [integration tests](./deployment/testing.md) against a
+deployment. It is recommended that this is configured to run automatically for at least
+`aio` and `staging` environments. It should also be safe to run against `production` as
+long as the tests are configured to target an isolated project.
+
 A
 [sample GitLab CI/CD configuration](https://github.com/stackhpc/azimuth-config/tree/stable/.gitlab-ci.yml.sample)
 is provided that implements this workflow for GitLab-hosted repositories.