Merge branch 'stackhpc/2023.1' into firewall-config

Author: jovial

.automation

@@ -1,5 +1,5 @@
 # This file is a list of path filters for the PR workflow in .github/workflows/stackhpc-pull-request.yml.
-aio:
+aio: &aio
   - '.automation'
   - '.automation.conf/config.sh'
   - '.automation.conf/tempest/load-lists/default'
@@ -20,6 +20,11 @@ aio:
   - 'kayobe-env'
   - 'requirements.txt'
   - 'terraform/aio/**'
-check-tags:
+check-tags: &check-tags
   - '.github/workflows/stackhpc-check-tags.yml'
   - 'etc/kayobe/kolla-image-tags.yml'
+  - 'etc/kayobe/pulp.yml'
+  - 'tools/kolla-images.py'
+build-kayobe-image:
+  - *aio
+  - *check-tags

.github/path-filters.yml

@@ -0,0 +1,81 @@
+# Generate inputs for the reusable multinode.yml workflow.
+# The test scenario is randomly selected.
+# The inputs are printed to stdout in GitHub step output key=value format.
+
+from dataclasses import dataclass
+import random
+import typing as t
+
+
+@dataclass
+class OSRelease:
+    distribution: str
+    release: str
+    ssh_username: str
+
+
+@dataclass
+class OpenStackRelease:
+    version: str
+    previous_version: str
+    os_releases: t.List[OSRelease]
+
+
+@dataclass
+class Scenario:
+    openstack_release: OpenStackRelease
+    os_release: OSRelease
+    neutron_plugin: str
+    upgrade: bool
+
+
+ROCKY_9 = OSRelease("rocky", "9", "cloud-user")
+UBUNTU_JAMMY = OSRelease("ubuntu", "jammy", "ubuntu")
+# NOTE(upgrade): Add supported releases here.
+OPENSTACK_RELEASES = [
+    OpenStackRelease("2024.1", "2023.1", [ROCKY_9, UBUNTU_JAMMY]),
+    OpenStackRelease("2023.1", "zed", [ROCKY_9, UBUNTU_JAMMY]),
+]
+NEUTRON_PLUGINS = ["ovs", "ovn"]
+
+
+def main() -> None:
+    scenario = random_scenario()
+    inputs = generate_inputs(scenario)
+    for name, value in inputs.items():
+        write_output(name, value)
+
+
+def random_scenario() -> Scenario:
+    openstack_release = random.choice(OPENSTACK_RELEASES)
+    os_release = random.choice(openstack_release.os_releases)
+    neutron_plugin = random.choice(NEUTRON_PLUGINS)
+    upgrade = random.random() > 0.6
+    return Scenario(openstack_release, os_release, neutron_plugin, upgrade)
+
+
+def generate_inputs(scenario: Scenario) -> t.Dict[str, str]:
+    branch = get_branch(scenario.openstack_release.version)
+    previous_branch = get_branch(scenario.openstack_release.previous_version)
+    inputs = {
+        "os_distribution": scenario.os_release.distribution,
+        "os_release": scenario.os_release.release,
+        "ssh_username": scenario.os_release.ssh_username,
+        "neutron_plugin": scenario.neutron_plugin,
+        "upgrade": str(scenario.upgrade).lower(),
+        "stackhpc_kayobe_config_version": branch,
+        "stackhpc_kayobe_config_previous_version": previous_branch,
+    }
+    return inputs
+
+
+def get_branch(version: str) -> str:
+    return f"stackhpc/{version}"
+
+
+def write_output(name: str, value: str) -> None:
+    print(f"{name}={value}")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/multinode-inputs.py

@@ -198,7 +198,7 @@ jobs:
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe seed host command run \
-          --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install zstd debootstrap kpartx cloud-init" --show-output
+          --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install cloud-init debootstrap git kpartx zstd" --show-output
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 

.github/workflows/overcloud-host-image-build.yml

@@ -167,7 +167,7 @@ jobs:
           VM_NETWORK: ${{ inputs.vm_network }}
           VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
-          VM_VOLUME_SIZE: ${{ inputs.upgrade && '45' || '35' }}
+          VM_VOLUME_SIZE: ${{ inputs.upgrade && '55' || '40' }}
           VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]'
 
       - name: Terraform Plan
@@ -179,6 +179,7 @@ jobs:
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
       - name: Terraform Apply
+        id: tf_apply
         run: |
           for attempt in $(seq 5); do
               if terraform apply -auto-approve; then
@@ -355,28 +356,82 @@ jobs:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
         if: inputs.upgrade
 
+      - name: Ensure we have IP on breth1 to reach the instances
+        # NOTE(wszumski): Whilst we don't need to create resources again, in some circumstances
+        # we can lose the IP address that allows us to connect to the instances. This playbook
+        # also fixes that issue.
+        run: |
+          docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: inputs.upgrade
+
       - name: Tempest tests
+        id: tempest
         run: |
           mkdir -p tempest-artifacts
           docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
             -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
-            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack -e rally_no_sensitive_log=false
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
+      - name: StackHPC OpenStack tests
+        id: stackhpc-openstack-tests
+        continue-on-error: true
+        run: |
+          mkdir -p sot-results
+          docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -v $(pwd)/sot-results:/stack/sot-results \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-openstack-tests.yml'
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
+      - name: Collect diagnostic information
+        id: diagnostics
+        run: |
+          mkdir -p diagnostics
+          sudo -E docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -v $(pwd)/diagnostics:/stack/diagnostics \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/diagnostics.yml'
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: ${{ !cancelled() && steps.tf_apply.outcome == 'success' }}
 
       - name: Upload test result artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: tempest-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' }}
-          path: tempest-artifacts/*
+          name: test-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' || '' }}
+          path: |
+            diagnostics/
+            tempest-artifacts/
+            sot-results/
+        if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-openstack-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }}
 
       - name: Fail if any Tempest tests failed
         run: |
           test $(wc -l < tempest-artifacts/failed-tests) -lt 1
 
+      - name: Fail if any StackHPC OpenStack tests failed
+        run: |
+          echo "Some StackHPC OpenStack tests failed."
+          echo "See HTML results artifact (sot-results) for details."
+          exit 1
+        if: steps.stackhpc-openstack-tests.outcome == 'failure'
+
       - name: Destroy
         run: terraform destroy -auto-approve
         working-directory: ${{ github.workspace }}/terraform/aio

.github/workflows/stackhpc-all-in-one.yml

@@ -46,6 +46,15 @@ jobs:
         run: |
           docker image pull $KAYOBE_IMAGE
 
+      - name: Check kolla-images.py image map and tag hierarchy
+        run: |
+          docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh \
+            '$KAYOBE_CONFIG_PATH/ansible/check-kolla-images-py.yml'
+
       - name: Check container image tags
         run: |
           docker run -t --rm \

.github/workflows/stackhpc-check-tags.yml

@@ -34,11 +34,10 @@ on:
         required: false
         default: true
       push-dirty:
-        description: Push scanned images that have vulnerabilities?
+        description: Push scanned images that have critical vulnerabilities?
         type: boolean
         required: false
-        # NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures
-        default: true
+        default: false
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -136,6 +135,10 @@ jobs:
         run: |
           curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0
 
+      - name: Install yq
+        run: |
+          curl -sL https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_amd64.tar.gz | tar xz && sudo mv yq_linux_amd64 /usr/bin/yq
+
       - name: Install Kayobe
         run: |
           mkdir -p venvs &&
@@ -176,7 +179,7 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Create build logs output directory
-        run: mkdir image-build-logs 
+        run: mkdir image-build-logs
 
       - name: Build kolla overcloud images
         id: build_overcloud_images
@@ -228,16 +231,23 @@ jobs:
         run: mv image-scan-output image-build-logs/image-scan-output
 
       - name: Fail if no images have passed scanning
-        run: if [ $(wc -l < image-build-logs/image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi
+        run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi
         if: ${{ !inputs.push-dirty }}
 
       - name: Copy clean images to push-attempt-images list
         run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
         if: inputs.push
 
+      # NOTE(seunghun1ee): This always appends dirty images with CVEs severity lower than critical.
+      # This should be reverted when it's decided to filter high level CVEs as well.
       - name: Append dirty images to push list
         run: |
           cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
+        if: ${{ inputs.push }}
+
+      - name: Append images with critical vulnerabilities to push list
+        run: |
+          cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt
         if: ${{ inputs.push && inputs.push-dirty }}
 
       - name: Push images
@@ -249,7 +259,7 @@ jobs:
 
           while read -r image; do
             # Retries!
-            for i in {1..5}; do 
+            for i in {1..5}; do
               if docker push $image; then
                 echo "Pushed $image"
                 break
@@ -283,8 +293,15 @@ jobs:
         run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
         if: ${{ !cancelled() }}
 
-      - name: Fail when images failed scanning
-        run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi
+      # NOTE(seunghun1ee): Currently we want to mark the job fail only when critical CVEs are detected.
+      # This can be used again instead of "Fail when critical vulnerabilities are found" when it's
+      # decided to fail the job on detecting high CVEs as well.
+      # - name: Fail when images failed scanning
+      #   run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi
+      #   if: ${{ !inputs.push-dirty && !cancelled() }}
+
+      - name: Fail when critical vulnerabilities are found
+        run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi
         if: ${{ !inputs.push-dirty && !cancelled() }}
 
       # NOTE(mgoddard): Trigger another CI workflow in the

.github/workflows/stackhpc-container-image-build.yml

@@ -0,0 +1,50 @@
+---
+# This workflow provides a periodic deploy of a multi-node test cluster.
+# The test scenario is randomly selected.
+
+name: Multinode periodic
+'on':
+  schedule:
+    # Runs nightly at 2:42 AM.
+    - cron: "42 2 * * *"
+jobs:
+  generate-inputs:
+    name: Generate inputs
+    runs-on: ubuntu-latest
+    outputs:
+      os_distribution: ${{ steps.generate-inputs.outputs.os_distribution }}
+      os_release: ${{ steps.generate-inputs.outputs.os_release }}
+      ssh_username: ${{ steps.generate-inputs.outputs.ssh_username }}
+      neutron_plugin: ${{ steps.generate-inputs.outputs.neutron_plugin }}
+      upgrade: ${{ steps.generate-inputs.outputs.upgrade }}
+      stackhpc_kayobe_config_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_version }}
+      stackhpc_kayobe_config_previous_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Generate inputs for multinode workflow
+        id: generate-inputs
+        run: |
+          python3 .github/workflows/multinode-inputs.py >> $GITHUB_OUTPUT
+
+      - name: Display generated inputs
+        run: |
+          echo '${{ toJSON(steps.generate-inputs.outputs) }}'
+  multinode:
+    name: Multinode periodic
+    needs:
+      - generate-inputs
+    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/[email protected]
+    with:
+      multinode_name: mn-prdc-${{ github.run_id }}
+      os_distribution: ${{ needs.generate-inputs.outputs.os_distribution }}
+      os_release: ${{ needs.generate-inputs.outputs.os_release }}
+      ssh_username: ${{ needs.generate-inputs.outputs.ssh_username }}
+      neutron_plugin: ${{ needs.generate-inputs.outputs.neutron_plugin }}
+      upgrade: ${{ needs.generate-inputs.outputs.upgrade == 'true' }}
+      stackhpc_kayobe_config_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_version }}
+      stackhpc_kayobe_config_previous_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }}
+      enable_slack_alert: true
+    secrets: inherit
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'