Merge pull request #389 from stackhpc/xena-wallaby-merge

markgoddard · web-flow · commit cd566b4b59b8 · 2023-02-13T15:11:52.000Z
xena: wallaby merge
diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
@@ -64,12 +64,12 @@ jobs:
       KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
       KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
            submodules: true
 
       - name: Install terraform
-        uses: hashicorp/setup-terraform@v1
+        uses: hashicorp/setup-terraform@v2
 
       - name: Initialise terraform
         run: terraform init
@@ -167,16 +167,14 @@ jobs:
           ENABLE_OVN: ${{ inputs.neutron_plugin == 'ovn' }}
           OS_DISTRIBUTION: ${{ inputs.os_distribution }}
 
-      # https://renehernandez.io/snippets/multiline-strings-as-a-job-output-in-github-actions/
+      # Use a heredoc to define a multiline string output
+      # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
       - name: Set SSH key output
         id: ssh_key
         run: |
-          ssh_key="$(cat terraform/aio/id_rsa)"
-          ssh_key="${ssh_key//'%'/'%25'}"
-          ssh_key="${ssh_key//$'\n'/'%0A'}"
-          ssh_key="${ssh_key//$'\r'/'%0D'}"
-          echo "::add-mask::$ssh_key"
-          echo "::set-output name=ssh_key::$ssh_key"
+          echo "ssh_key<<EOF" >> $GITHUB_OUTPUT
+          cat terraform/aio/id_rsa >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
 
       # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
       - name: Pull latest Kayobe image
diff --git a/.github/workflows/stackhpc-build-kayobe-image.yml b/.github/workflows/stackhpc-build-kayobe-image.yml
@@ -37,20 +37,20 @@ jobs:
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - name: Checkout kayobe config
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
            submodules: true
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v4
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
@@ -167,10 +167,15 @@ jobs:
       # stackhpc-release-train repository.
       - name: Trigger container image repository sync
         run: |
+          filter='${{ inputs.regexes }}'
+          if [[ -n $filter ]] && [[ ${{ github.event.inputs.seed }} == 'true' ]]; then
+            filter="$filter bifrost"
+          fi
           gh workflow run \
           container-sync.yml \
           --repo stackhpc/stackhpc-release-train \
-          --ref main
+          --ref main \
+          -f filter="$filter"
         env:
           GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
 
diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml
@@ -28,7 +28,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup Python ${{ matrix.python-version }} 🐍
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Tox 📦
diff --git a/etc/kayobe/containers/pulp/post.yml b/etc/kayobe/containers/pulp/post.yml
@@ -17,3 +17,13 @@
   register: pulp_manager_result
   failed_when:
     - "'Successfully set password' not in pulp_manager_result.stdout"
+
+- name: Manage Pulp content checksums
+  become: true
+  command: >-
+    docker exec -u root pulp
+    bash -c
+      'pulpcore-manager handle-artifact-checksums'
+  when:
+    - stackhpc_pulp_sync_for_local_container_build | bool
+    - pulp_settings.changed
diff --git a/etc/kayobe/containers/pulp/pre.yml b/etc/kayobe/containers/pulp/pre.yml
@@ -19,6 +19,7 @@
     dest: /opt/kayobe/containers/pulp/settings.py
     mode: 0644
   become: true
+  register: pulp_settings
 
 - name: Configure TLS for local Pulp
   when: pulp_enable_tls | bool
diff --git a/etc/kayobe/containers/pulp/settings.py b/etc/kayobe/containers/pulp/settings.py
@@ -2,3 +2,7 @@
 ANSIBLE_API_HOSTNAME='{{ pulp_url }}'
 ANSIBLE_CONTENT_HOSTNAME='{{ pulp_url }}/pulp/content'
 TOKEN_AUTH_DISABLED=True
+{% if stackhpc_pulp_sync_for_local_container_build | bool %}
+ALLOWED_CONTENT_CHECKSUMS = ["sha1", "sha224", "sha256", "sha384", "sha512"]
+{% endif %}
+
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
@@ -35,13 +35,18 @@ pulp_proxy_url: "{{ omit }}"
 # Base URL of the StackHPC Pulp service.
 stackhpc_release_pulp_url: "https://ark.stackhpc.com"
 
-# Credentials used to access the StackHPC Ark pulp server.
+# Credentials used to access the StackHPC Pulp service.
 stackhpc_release_pulp_username:
 stackhpc_release_pulp_password:
 
 # Content URL of the StackHPC Pulp service.
 stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content"
 
+# Sync all repositories required for building Kolla images from the
+# StackHPC Ark Pulp service to local Pulp.
+# NOTE: Only RPM repositories are supported.
+stackhpc_pulp_sync_for_local_container_build: false
+
 ###############################################################################
 # Debs
 
@@ -333,6 +338,128 @@ stackhpc_pulp_repository_rpm_repos:
     state: present
     required: "{{ stackhpc_pulp_sync_el_8 }}"
 
+  # Additional repositories required to build Kolla containers from local Pulp service.
+  - name: CentOS Stream 8 - OpenStack Xena
+    url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/cloud/x86_64/openstack-xena/{{ stackhpc_pulp_repo_centos_stream_8_openstack_xena_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
+    client_cert: ""
+    client_key: ""
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
+  - name: CentOS Stream 8 - Advanced Virtualization
+    url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/virt/x86_64/advancedvirt-common/{{ stackhpc_pulp_repo_centos_stream_8_advanced_virtualization_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
+    client_cert: ""
+    client_key: ""
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
+  - name: CentOS Stream 8 - NFV OpenvSwitch
+    url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/nfv/x86_64/openvswitch-2/{{ stackhpc_pulp_repo_centos_stream_8_nfv_openvswitch_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
+    client_cert: ""
+    client_key: ""
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
+  - name: CentOS Stream 8 - PowerTools
+    url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/PowerTools/x86_64/os/{{ stackhpc_pulp_repo_centos_stream_8_powertools_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
+    client_cert: ""
+    client_key: ""
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
+  - name: CentOS 8 - OpsTools - collectd
+    url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/opstools/x86_64/collectd-5/{{ stackhpc_pulp_repo_centos_stream_8_opstools_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
+    client_cert: ""
+    client_key: ""
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
+  - name: CentOS Stream 8 - Ceph Pacific
+    url: "{{ stackhpc_release_pulp_content_url }}/centos/8-stream/storage/x86_64/ceph-pacific/{{ stackhpc_pulp_repo_centos_stream_8_storage_ceph_pacific_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
+    client_cert: ""
+    client_key: ""
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
+  - name: ELK repository for 7.x packages
+    url: "{{ stackhpc_release_pulp_content_url }}/elasticsearch/oss-7.x/{{ stackhpc_pulp_repo_elasticsearch_logstash_kibana_7_x_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
+    client_cert: ""
+    client_key: ""
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
+  - name: Grafana
+    url: "{{ stackhpc_release_pulp_content_url }}/grafana/oss/rpm/{{ stackhpc_pulp_repo_grafana_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
+    client_cert: ""
+    client_key: ""
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
+  - name: RabbitMQ - Erlang
+    url: "{{ stackhpc_release_pulp_content_url }}/rabbitmq/erlang/el/8/x86_64/{{ stackhpc_pulp_repo_rabbitmq_erlang_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
+    client_cert: ""
+    client_key: ""
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
+  - name: RabbitMQ - Server
+    url: "{{ stackhpc_release_pulp_content_url }}/rabbitmq/rabbitmq-server/el/8/x86_64/{{ stackhpc_pulp_repo_rabbitmq_server_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
+    client_cert: ""
+    client_key: ""
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
+  - name: TreasureData 4
+    url: "{{ stackhpc_release_pulp_content_url }}/treasuredata/4/redhat/8/x86_64/{{ stackhpc_pulp_repo_treasuredata_4_version }}"
+    remote_username: "{{ stackhpc_release_pulp_username }}"
+    remote_password: "{{ stackhpc_release_pulp_password }}"
+    client_cert: ""
+    client_key: ""
+    policy: on_demand
+    sync_policy: mirror_complete
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
 # Publication format is a subset of distribution.
 stackhpc_pulp_publication_rpm_development: "{{ stackhpc_pulp_distribution_rpm_development }}"
 
@@ -404,6 +531,63 @@ stackhpc_pulp_distribution_rpm_development:
     state: present
     required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
 
+  # Additional repositories required to build Kolla containers from local Pulp service.
+  - name: "centos-stream-8-openstack-xena-development"
+    base_path: "centos/8-stream/cloud/x86_64/openstack-xena/development"
+    repository: CentOS Stream 8 - OpenStack Xena
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "centos-stream-8-advanced-virtualization-development"
+    base_path: "centos/8-stream/virt/x86_64/advancedvirt-common/development"
+    repository: CentOS Stream 8 - Advanced Virtualization
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "centos-stream-8-nfv-openvswitch-development"
+    base_path: "centos/8-stream/nfv/x86_64/openvswitch-2/development"
+    repository: CentOS Stream 8 - NFV OpenvSwitch
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "centos-stream-8-powertools-development"
+    base_path: "centos/8-stream/PowerTools/x86_64/os/development"
+    repository: CentOS Stream 8 - PowerTools
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "centos-8-opstools-collectd-development"
+    base_path: "centos/8-stream/opstools/x86_64/collectd-5/development"
+    repository: CentOS 8 - OpsTools - collectd
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "centos-stream-8-ceph-pacific-development"
+    base_path: "centos/8-stream/storage/x86_64/ceph-pacific/development"
+    repository: CentOS Stream 8 - Ceph Pacific
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "elasticsearch-logstash-kibana-7.x-development"
+    base_path: "elasticsearch/oss-7.x/development"
+    repository: ELK repository for 7.x packages
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "grafana-development"
+    base_path: "grafana/oss/rpm/development"
+    repository: Grafana
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "rabbitmq-erlang-development"
+    base_path: "rabbitmq/erlang/el/8/x86_64/development"
+    repository: RabbitMQ - Erlang
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "rabbitmq-server-development"
+    base_path: "rabbitmq/rabbitmq-server/el/8/x86_64/development"
+    repository: RabbitMQ - Server
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "treasuredata-4-development"
+    base_path: "treasuredata/4/redhat/8/x86_64/development"
+    repository: TreasureData 4
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
 # Development gets promoted to production.
 stackhpc_pulp_distribution_rpm_production:
   # Base CentOS 8 Stream repositories
@@ -469,6 +653,63 @@ stackhpc_pulp_distribution_rpm_production:
     state: present
     required: "{{ stackhpc_pulp_sync_el_8 | bool }}"
 
+  # Additional repositories required to build Kolla containers from local Pulp service.
+  - name: "centos-stream-8-openstack-xena-production"
+    base_path: "centos/8-stream/cloud/x86_64/openstack-xena/production"
+    repository: CentOS Stream 8 - OpenStack Xena
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "centos-stream-8-advanced-virtualization-production"
+    base_path: "centos/8-stream/virt/x86_64/advancedvirt-common/production"
+    repository: CentOS Stream 8 - Advanced Virtualization
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "centos-stream-8-nfv-openvswitch-production"
+    base_path: "centos/8-stream/nfv/x86_64/openvswitch-2/production"
+    repository: CentOS Stream 8 - NFV OpenvSwitch
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "centos-stream-8-powertools-production"
+    base_path: "centos/8-stream/PowerTools/x86_64/os/production"
+    repository: CentOS Stream 8 - PowerTools
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "centos-8-opstools-collectd-production"
+    base_path: "centos/8-stream/opstools/x86_64/collectd-5/production"
+    repository: CentOS 8 - OpsTools - collectd
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "centos-stream-8-ceph-pacific-production"
+    base_path: "centos/8-stream/storage/x86_64/ceph-pacific/production"
+    repository: CentOS Stream 8 - Ceph Pacific
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "elasticsearch-logstash-kibana-7.x-production"
+    base_path: "elasticsearch/oss-7.x/production"
+    repository: ELK repository for 7.x packages
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "grafana-production"
+    base_path: "grafana/oss/rpm/production"
+    repository: Grafana
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "rabbitmq-erlang-production"
+    base_path: "rabbitmq/erlang/el/8/x86_64/production"
+    repository: RabbitMQ - Erlang
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "rabbitmq-server-production"
+    base_path: "rabbitmq/rabbitmq-server/el/8/x86_64/production"
+    repository: RabbitMQ - Server
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+  - name: "treasuredata-4-production"
+    base_path: "treasuredata/4/redhat/8/x86_64/production"
+    repository: TreasureData 4
+    state: present
+    required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_8 | bool }}"
+
 ###############################################################################
 # Containers
 
diff --git a/releasenotes/notes/kolla-local-builds-eca0277a969803fd.yaml b/releasenotes/notes/kolla-local-builds-eca0277a969803fd.yaml
