Merge pull request #12 from stackhpc/2024.9.1-sync

2024.9.1 sync

Author: MoteHue

.github-deploy-prod.yml.sample

@@ -50,4 +50,4 @@ jobs:
           ./bin/ensure-venv
           source ./bin/activate  ${{ inputs.environment }}
           ansible-galaxy install -fr ./requirements.yml
-          ansible-playbook stackhpc.azimuth_ops.provision
+          ansible-playbook azimuth_cloud.azimuth_ops.provision

.github-deploy-staging.yml.sample

@@ -37,4 +37,4 @@ jobs:
           ./bin/ensure-venv
           source ./bin/activate  ${{ env.azimuth-config-env-name }}
           ansible-galaxy install -fr ./requirements.yml
-          ansible-playbook stackhpc.azimuth_ops.provision
+          ansible-playbook azimuth_cloud.azimuth_ops.provision

.github-upgrade-check.yml.sample

@@ -0,0 +1,68 @@
+# This workflow compares a downstream azimuth-config repository for a specific site with the upstream
+# stackhpc/azimuth-config repository to check whether there is a new upstream version available. If a
+# newer tag is found in the upstream repository then a pull request is created to the downstream repo
+# in order to merge in the changes from the new upstream release.
+
+# To use this workflow in a downstream azimuth-config repository simply copy it into .github/workflows
+# and give it an appropriate name, e.g.
+# cp .github-upgrade-check.yml.sample .github/workflows/upgrade-check.yml
+
+name: Check for upstream updates
+on:
+  schedule:
+    - cron: "0 9 * * *"
+  workflow_dispatch:
+jobs:
+  check_for_update:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+
+      - name: Checkout the config repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      # Based on equivalent GitLab CI job
+      - name: Check for new release
+        shell: bash
+        run: |
+          set -xe
+
+          # Install dependency
+          apt update && apt install -y git-crypt
+
+          # Tell git who we are for commits
+          git config user.email "${{ github.actor }}[email protected]"
+          git config user.name "${{ github.actor }} CI"
+
+          # Create the merge branch and write vars to .mergeenv file
+          ./bin/create-merge-branch
+
+      - name: Set release tag output
+        id: release_tag
+        if: ${{ hashFiles('.mergeenv') }}
+        run: source .mergeenv && echo value=$RELEASE_TAG >> $GITHUB_OUTPUT
+
+      - name: Set branch name output
+        id: branch_name
+        if: ${{ hashFiles('.mergeenv') }}
+        run: source .mergeenv && echo value=$BRANCH_NAME >> $GITHUB_OUTPUT
+
+      - name: Remove tmp file
+        run: rm .mergeenv
+        if: ${{ hashFiles('.mergeenv') }}
+
+      - name: Create Pull Request
+        if: ${{ steps.release_tag.outputs.value }}
+        uses: peter-evans/create-pull-request@v6
+        with:
+          base: main
+          branch: ${{ steps.branch_name.outputs.value }}
+          title: "Upgrade Azimuth to ${{ steps.release_tag.outputs.value }}"
+          body: This PR was automatically generated by GitHub Actions.
+          commit-message: "Upgrade Azimuth to ${{ steps.release_tag.outputs.value }}"
+          delete-branch: true

.github/actions/destroy/action.yml

@@ -9,7 +9,7 @@ runs:
         set -e
         source ./ci.env
         source ./bin/activate "$AZIMUTH_CONFIG_ENVIRONMENT" "$AZIMUTH_ENVIRONMENT"
-        ansible-playbook stackhpc.azimuth_ops.destroy \
+        ansible-playbook azimuth_cloud.azimuth_ops.destroy \
           -e @extra-vars.yml \
           -e force_destroy=true \
           -e capi_cluster_volumes_policy=delete
@@ -50,7 +50,7 @@ runs:
       if: ${{ always() }}
 
     - name: Release S3 lock
-      uses: stackhpc/github-actions/s3-lock@master
+      uses: azimuth-cloud/github-actions/s3-lock@master
       with:
         host: ${{ steps.s3-lock-config.outputs.host }}
         access-key: ${{ steps.s3-lock-config.outputs.access-key }}

.github/actions/provision/action.yml

@@ -10,6 +10,4 @@ runs:
         set -e
         source ./ci.env
         source ./bin/activate "$AZIMUTH_CONFIG_ENVIRONMENT" "$AZIMUTH_ENVIRONMENT"
-        ansible-playbook stackhpc.azimuth_ops.provision -e @extra-vars.yml
-      env:
-        ANSIBLE_CALLBACKS_ENABLED: ansible.posix.profile_tasks
+        ansible-playbook azimuth_cloud.azimuth_ops.provision -e @extra-vars.yml

.github/actions/release-notes/release-notes.py

@@ -38,6 +38,11 @@
         "path": "roles/azimuth_identity_operator/defaults/main.yml",
         "version_key": "azimuth_identity_operator_chart_version",
     },
+    {
+        "name": "azimuth-schedule-operator",
+        "path": "roles/azimuth_schedule_operator/defaults/main.yml",
+        "version_key": "azimuth_schedule_operator_chart_version",
+    },
     {
         "name": "zenith",
         "path": "roles/zenith/defaults/main.yml",
@@ -61,20 +66,30 @@
     {
         "name": "caas-workstation",
         "path": "roles/azimuth_caas_operator/defaults/main.yml",
-        "version_key": "azimuth_caas_stackhpc_workstation_git_version",
+        "version_key": [
+            "azimuth_caas_workstation_default_git_version",
+            "azimuth_caas_stackhpc_workstation_git_version",
+        ],
     },
     {
         "name": "caas-repo2docker",
         "path": "roles/azimuth_caas_operator/defaults/main.yml",
-        "version_key": "azimuth_caas_stackhpc_repo2docker_git_version",
+        "version_key": [
+            "azimuth_caas_repo2docker_default_git_version",
+            "azimuth_caas_stackhpc_repo2docker_git_version",
+        ],
     },
     {
         "name": "caas-r-studio-server",
         "path": "roles/azimuth_caas_operator/defaults/main.yml",
-        "version_key": "azimuth_caas_stackhpc_rstudio_git_version",
+        "version_key": [
+            "azimuth_caas_rstudio_default_git_version",
+            "azimuth_caas_stackhpc_rstudio_git_version",
+        ],
     },
     {
         "name": "ansible-slurm-appliance",
+        "org": "stackhpc",
         "path": "roles/azimuth_caas_operator/defaults/main.yml",
         "version_key": "azimuth_caas_stackhpc_slurm_appliance_git_version",
     },
@@ -174,24 +189,31 @@ def fetch_component_version_for_ops_tag(session, tag, component):
     Returns the version of the specified component that is used in the specified azimuth-ops tag.
     """
     response = session.get(
-        f"{API_URL}/repos/stackhpc/ansible-collection-azimuth-ops/contents/{component['path']}",
+        f"{API_URL}/repos/azimuth-cloud/ansible-collection-azimuth-ops/contents/{component['path']}",
         params = { "ref": tag },
         headers = { "Content-Type": "application/vnd.github.raw+json" }
     )
     response.raise_for_status()
     content = base64.b64decode(response.json()["content"])
-    return yaml.safe_load(content)[component["version_key"]]
+    data = yaml.safe_load(content)
+    # In order to allow version keys to change between azimuth-ops versions, we support
+    # specifying a list of keys which we try in order
+    if isinstance(component["version_key"], list):
+        version_keys = component["version_key"]
+    else:
+        version_keys = [component["version_key"]]
+    return next(data[key] for key in version_keys if key in data)
 
 
-def release_notes_for_component(session, name, from_version, to_version):
+def release_notes_for_component(session, name, org, from_version, to_version):
     """
     Produces the release notes for a component between the specified versions.
     """
     print(f"[INFO]   collecting release notes for {name}")
     release_notes = []
     for release in fetch_releases(
         session,
-        f"stackhpc/{name}",
+        f"{org}/{name}",
         min = from_version,
         inclusive_min = False,
         max = to_version,
@@ -228,11 +250,14 @@ def main():
     parser.add_argument(
         "--repo",
         help = "The config repository to target.",
-        default = "stackhpc/azimuth-config"
+        default = "azimuth-cloud/azimuth-config"
     )
     parser.add_argument("tag", help = "The tag to generate release notes for.")
     args = parser.parse_args()
 
+    # Make sure that the YAML SafeLoader respects Ansible's !unsafe tag
+    yaml.SafeLoader.add_constructor("!unsafe", yaml.SafeLoader.construct_scalar)
+
     session = github_session(args.token)
 
     print(f"[INFO] fetching release for tag - {args.tag}")
@@ -267,6 +292,7 @@ def main():
             release_notes_for_component(
                 session,
                 "ansible-collection-azimuth-ops",
+                "azimuth-cloud",
                 previous_ops_tag,
                 current_ops_tag
             )
@@ -290,6 +316,7 @@ def main():
                 release_notes_for_component(
                     session,
                     component['name'],
+                    component.get('org', 'azimuth-cloud'),
                     component_vn_previous,
                     component_vn_current
                 )

.github/actions/setup/action.yml

@@ -4,7 +4,7 @@ inputs:
   repository:
     description: The repository to use for the Azimuth configuration.
     required: true
-    default: stackhpc/azimuth-config
+    default: azimuth-cloud/azimuth-config
   ref:
     description: The ref to use for the Azimuth configuration.
     required: true
@@ -75,7 +75,7 @@ runs:
         REQUIREMENTS_CONTENT: |
           ---
           collections:
-            - name: https://github.com/stackhpc/ansible-collection-azimuth-ops.git
+            - name: https://github.com/azimuth-cloud/ansible-collection-azimuth-ops.git
               type: git
               version: ${{ inputs.azimuth-ops-version }}
       if: ${{ inputs.azimuth-ops-version != '' }}
@@ -134,7 +134,7 @@ runs:
         echo "bucket=${CI_S3_LOCK_BUCKET}" >> "$GITHUB_OUTPUT"
 
     - name: Acquire S3 lock
-      uses: stackhpc/github-actions/s3-lock@master
+      uses: azimuth-cloud/github-actions/s3-lock@master
       with:
         host: ${{ steps.s3-lock-config.outputs.host }}
         access-key: ${{ steps.s3-lock-config.outputs.access-key }}
@@ -153,30 +153,43 @@ runs:
         ansible_var() {
             ANSIBLE_LOAD_CALLBACK_PLUGINS=true \
             ANSIBLE_STDOUT_CALLBACK=json \
+            ANSIBLE_JSON_INDENT=0 \
             ansible -m debug -a "var=$1" -e @extra-vars.yml all | \
-              jq -r ".plays[0].tasks[0].hosts.localhost.$1"
+               jq -r -R "fromjson? | .plays[0].tasks[0].hosts.localhost.$1"
         }
 
         EXTNET_ID="$(ansible_var infra_external_network_id)"
         INSTALL_MODE="$(ansible_var install_mode)"
 
-        INGRESS_IP="$(
-          openstack floating ip create $EXTNET_ID \
-            --description "ingress IP for $AZIMUTH_ENVIRONMENT" \
-            --format value \
-            --column floating_ip_address
-        )"
+        until \
+          INGRESS_IP="$(
+            openstack floating ip create $EXTNET_ID \
+              --description "ingress IP for $AZIMUTH_ENVIRONMENT" \
+              --format value \
+              --column floating_ip_address
+          )"
+        do
+            echo "waiting for ingress IP to become available"
+            sleep 60
+        done
+        echo "assigned $INGRESS_IP for ingress"
         cat >> ci.env <<EOF
         export INGRESS_IP="$INGRESS_IP"
         EOF
 
         if [ "$INSTALL_MODE" = "ha" ]; then
-            ZENITH_SSHD_IP="$(
-              openstack floating ip create $EXTNET_ID \
-                --description "Zenith SSHD IP for $AZIMUTH_ENVIRONMENT" \
-                --format value \
-                --column floating_ip_address
-            )"
+            until \
+              ZENITH_SSHD_IP="$(
+                openstack floating ip create $EXTNET_ID \
+                  --description "Zenith SSHD IP for $AZIMUTH_ENVIRONMENT" \
+                  --format value \
+                  --column floating_ip_address
+              )"
+            do
+                echo "waiting for Zenith SSHD IP to become available"
+                sleep 60
+            done
+            echo "assigned $ZENITH_SSHD_IP for Zenith SSHD"
             cat >> ci.env <<EOF
         export ZENITH_SSHD_IP="$ZENITH_SSHD_IP"
         EOF

.github/actions/test/action.yml

@@ -28,7 +28,7 @@ runs:
         set -e
         source ./ci.env
         source ./bin/activate "$AZIMUTH_CONFIG_ENVIRONMENT" "$AZIMUTH_ENVIRONMENT"
-        ansible-playbook stackhpc.azimuth_ops.generate_tests -e @extra-vars.yml
+        ansible-playbook azimuth_cloud.azimuth_ops.generate_tests -e @extra-vars.yml
 
     - name: Run test suite
       shell: bash

.github/environments/arcus-ha/ansible.cfg

@@ -2,8 +2,14 @@
 inventory = ../../../environments/base/inventory,../../../environments/ha/inventory,../../../environments/demo/inventory,../common/inventory,../arcus/inventory,./inventory
 roles_path = ../../../.ansible/roles
 collections_path = ../../../.ansible/collections
+stdout_callback = yaml
+bin_ansible_callbacks = True
+callbacks_enabled = ansible.posix.profile_tasks
 
+# Disable host key checking as hosts are dynamically replaced
 host_key_checking = False
 
 [ssh_connection]
+pipelining = True
+ssh_extra_args = -o ControlPersist=1h
 retries = 3

.github/environments/arcus/ansible.cfg

@@ -2,8 +2,14 @@
 inventory = ../../../environments/base/inventory,../../../environments/singlenode/inventory,../../../environments/demo/inventory,../common/inventory,./inventory
 roles_path = ../../../.ansible/roles
 collections_path = ../../../.ansible/collections
+stdout_callback = yaml
+bin_ansible_callbacks = True
+callbacks_enabled = ansible.posix.profile_tasks
 
+# Disable host key checking as hosts are dynamically replaced
 host_key_checking = False
 
 [ssh_connection]
+pipelining = True
+ssh_extra_args = -o ControlPersist=1h
 retries = 3