Add upgrade tests for platforms (azimuth-cloud#164)

* Add upgrade tests for platforms

* azimuth-ops changes merged and tagged

Author: Matt Pryor

.github/workflows/test-upgrade.yml

@@ -66,7 +66,9 @@ jobs:
           set -e
           source ./ci.env
           source ./bin/activate "$AZIMUTH_CONFIG_ENVIRONMENT" "$AZIMUTH_ENVIRONMENT"
-          ansible-playbook stackhpc.azimuth_ops.generate_tests -e @extra-vars.yml
+          ansible-playbook stackhpc.azimuth_ops.generate_tests \
+            -e @extra-vars.yml \
+            -e generate_tests_include_upgrade_tests=true
 
       - name: Create test platforms
         id: tests-create
@@ -115,17 +117,40 @@ jobs:
         uses: ./.actions/.github/actions/provision
         if: ${{ !cancelled() && steps.update-ansible.conclusion == 'success' }}
 
-      - name: Verify test platforms post upgrade
+      - name: Regenerate test suite
+        id: regenerate-tests
         run: |
           set -e
           source ./ci.env
           source ./bin/activate "$AZIMUTH_CONFIG_ENVIRONMENT" "$AZIMUTH_ENVIRONMENT"
-          ./.actions/bin/run-tests --include verify --outputdir reports/verify-post-upgrade
+          ansible-playbook stackhpc.azimuth_ops.generate_tests \
+            -e @extra-vars.yml \
+            -e generate_tests_include_upgrade_tests=true
         if: ${{ !cancelled() && steps.upgrade-azimuth.conclusion == 'success' }}
 
-      # TODO(mkjpryor) patch/upgrade not implemented in test generation yet
-      # - name: Upgrade test platforms
-      # - name: Verify test platforms
+      - name: Verify test platforms post Azimuth upgrade
+        run: |
+          set -e
+          source ./ci.env
+          source ./bin/activate "$AZIMUTH_CONFIG_ENVIRONMENT" "$AZIMUTH_ENVIRONMENT"
+          ./.actions/bin/run-tests --include verify --outputdir reports/verify-post-azimuth-upgrade
+        if: ${{ !cancelled() && steps.regenerate-tests.conclusion == 'success' }}
+
+      - name: Upgrade test platforms
+        run: |
+          set -e
+          source ./ci.env
+          source ./bin/activate "$AZIMUTH_CONFIG_ENVIRONMENT" "$AZIMUTH_ENVIRONMENT"
+          ./.actions/bin/run-tests --include upgrade --outputdir reports/upgrade
+        if: ${{ !cancelled() && steps.regenerate-tests.conclusion == 'success' }}
+
+      - name: Verify test platforms post platform upgrade
+        run: |
+          set -e
+          source ./ci.env
+          source ./bin/activate "$AZIMUTH_CONFIG_ENVIRONMENT" "$AZIMUTH_ENVIRONMENT"
+          ./.actions/bin/run-tests --include verify --outputdir reports/verify-post-platform-upgrade
+        if: ${{ !cancelled() && steps.regenerate-tests.conclusion == 'success' }}
 
       - name: Delete test platforms
         run: |

bin/run-tests

@@ -23,7 +23,13 @@ ansible_variable() {
 
 # Get the test directory
 test_directory="$(ansible_variable generate_tests_suite_directory)"
+test_vars_file="$(ansible_variable generate_tests_vars_file)"
 # Use a headless browser by default
 export MOZ_HEADLESS="${MOZ_HEADLESS:-"1"}"
 # Execute the test suite, including any args we were given
-exec robot --loglevel debug --consolecolors on "$@" "$test_directory"
+exec robot \
+  --loglevel debug \
+  --consolecolors on \
+  --variablefile "$test_vars_file" \
+  "$@" \
+  "$test_directory"

environments/base/inventory/group_vars/all.yml

@@ -33,6 +33,7 @@ pgo_kustomization_directory: "{{ (work_directory, 'postgres-operator') | path_jo
 keycloak_operator_kustomization_directory: "{{ (work_directory, 'keycloak-operator') | path_join }}"
 awx_operator_directory: "{{ (work_directory, 'awx-operator') | path_join }}"
 # Use the configured work directory for the test suite
+generate_tests_vars_file: "{{ (work_directory, 'testvars.yaml') | path_join }}"
 generate_tests_suite_directory: "{{ (work_directory, 'test-suite') | path_join }}"
 
 # Enable cert-manager by default

requirements.yml

@@ -3,7 +3,7 @@
 collections:
   - name: https://github.com/stackhpc/ansible-collection-azimuth-ops.git
     type: git
-    version: 0.9.1
+    version: 0.9.2
   # For local development
   # - type: dir
   #   source: ../ansible-collection-azimuth-ops