Merge pull request #620 from stackhpc/yoga-zed-merge-2

Yoga zed merge v2

Author: markgoddard

.github/workflows/overcloud-host-image-build.yml

@@ -31,7 +31,7 @@ jobs:
   overcloud-host-image-build:
     name: Build overcloud host images
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder-rl9]
     permissions: {}
     steps:
       - uses: actions/checkout@v3
@@ -65,10 +65,9 @@ jobs:
           ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
           path: src/kayobe
 
-      # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed
       - name: Install dependencies
         run: |
-          sudo dnf -y install python3-virtualenv zstd
+          sudo dnf -y install zstd
 
       - name: Setup networking
         run: |

.github/workflows/overcloud-host-image-promote.yml

@@ -21,7 +21,7 @@ jobs:
   overcloud-host-image-promote:
     name: Promote overcloud host image
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder-rl9]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -40,11 +40,6 @@ jobs:
           ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
           path: src/kayobe
 
-      # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed
-      - name: Install dependencies
-        run: |
-          sudo dnf -y install python3-virtualenv
-
       - name: Setup networking
         run: |
           if ! ip l show breth1 >/dev/null 2>&1; then

.github/workflows/stackhpc-all-in-one.yml

@@ -79,7 +79,7 @@ jobs:
       - name: Output image tag
         id: image_tag
         run: |
-          echo image_tag=$(grep stackhpc_${{ inputs.os_distribution }}_${{ inputs.os_release }}_overcloud_host_image_version etc/kayobe/environments/ci-aio/stackhpc-ci.yml | awk '{print $2}') >> $GITHUB_OUTPUT
+          echo image_tag=$(grep stackhpc_${{ inputs.os_distribution }}_$(sed s/-/_/ <(echo "${{ inputs.os_release }}"))_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
 
       # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
       - name: Output image name

.github/workflows/stackhpc-container-image-build.yml

@@ -40,7 +40,7 @@ jobs:
   generate-tag:
     name: Generate container image tag
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder-rl9]
     permissions: {}
     outputs:
       datetime_tag: ${{ steps.datetime_tag.outputs.datetime_tag }}
@@ -88,7 +88,7 @@ jobs:
   container-image-build:
     name: Build Kolla container images
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder]
+    runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder-rl9]
     timeout-minutes: 720
     permissions: {}
     strategy:
@@ -107,11 +107,6 @@ jobs:
           ref: refs/heads/stackhpc/${{ needs.generate-tag.outputs.openstack_release }}
           path: src/kayobe
 
-      # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed
-      - name: Install dependencies
-        run: |
-          sudo dnf -y install python3-virtualenv
-
       - name: Setup networking
         run: |
           if ! ip l show breth1 >/dev/null 2>&1; then
@@ -127,14 +122,6 @@ jobs:
           sudo ip l set dummy1 up
           sudo ip l set dummy1 master breth1
 
-      # FIXME: Without this workaround we see the following issue after the runner is power cycled:
-      # TASK [MichaelRigart.interfaces : RedHat | ensure network service is started and enabled] ***
-      #   Unable to start service network: Job for network.service failed because the control process exited with error code.
-      #   See \"systemctl status network.service\" and \"journalctl -xe\" for details.
-      - name: Kill dhclient (workaround)
-        run: |
-          (sudo killall dhclient || true) && sudo systemctl restart network
-
       - name: Install Kayobe
         run: |
           mkdir -p venvs &&

.readthedocs.yaml

@@ -0,0 +1,27 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  apt_packages:
+    - tox
+  os: ubuntu-22.04
+  tools:
+    python: "3.7"
+  jobs:
+    post_checkout:
+      - git remote set-branches origin master stackhpc/zed stackhpc/yoga stackhpc/xena stackhpc/wallaby
+      - git fetch --unshallow
+
+# Build documentation in the doc/source/ directory with Sphinx
+sphinx:
+  configuration: doc/source/conf.py
+
+# We recommend specifying your dependencies to enable reproducible builds:
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: doc/requirements.txt

.yamllint

@@ -11,3 +11,12 @@ rules:
     require-starting-space: false
   line-length: disable
   truthy: disable
+
+ignore: |
+  *etc/kayobe/environments/ci-multinode/secrets.yml
+  *etc/kayobe/environments/ci-multinode/kolla/globals-tls-config.yml
+  .automation/
+  releasenotes/
+  .github/
+  .gitlab/
+  .gitlab-ci.yml

doc/source/configuration/host-images.rst

@@ -4,6 +4,42 @@
 Host Images
 ===========
 
+Pulling host images
+===================
+
+StackHPC provides pre-built overcloud host images through Ark, which can be
+consumed using the configuration provided by this repository.
+
+When configured, an image will be downloaded to the seed during the
+``kayobe seed service deploy`` step, and subsequently deployed using bifrost
+with ``kayobe overcloud provision``.
+
+To use these images, set ``stackhpc_download_overcloud_host_images`` to true
+in ``etc/kayobe/stackhpc-overcloud-host-images.yml``.
+
+Currently, images exist for the following operating systems:
+
+* Rocky Linux 9
+* Ubuntu Jammy 22.04
+
+The image to download is selected automatically using the ``os_distribution``
+and ``os_release`` variables. These images are versioned and a variable for
+each OS is stored in ``pulp-host-image-versions.yml``.
+
+This content requires the same set of credentials as is used for other
+release train content.
+
+The Ark pulp credentials issued by StackHPC should be configured in
+``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the password:
+
+.. code-block:: yaml
+
+   stackhpc_release_pulp_username: <username>
+   stackhpc_release_pulp_password: <password>
+
+Building host images
+====================
+
 StackHPC Kayobe configuration provides configuration for some standard
 overcloud host images, built using the :kayobe-doc:`overcloud DIB
 <configuration/reference/overcloud-dib.html>` functionality of Kayobe.

doc/source/configuration/release-train.rst

@@ -2,9 +2,9 @@
 StackHPC Release Train
 ======================
 
-StackHPC provides packages and container images for OpenStack via `Ark
-<https://ark.stackhpc.com>`__. These artifacts are built and released using a
-process known as the `Release Train
+StackHPC provides packages, container images, and host images for OpenStack via
+`Ark <https://ark.stackhpc.com>`__. These artifacts are built and released using
+a process known as the `Release Train
 <https://stackhpc.github.io/stackhpc-release-train/>`__.
 
 Deployments should use a local `Pulp <https://pulpproject.org/>`__ repository
@@ -41,6 +41,7 @@ This configuration provides the following:
 * Playbooks to synchronise a local Pulp service with Ark
 * Configuration to use the local Pulp repository mirrors on control plane hosts
 * Configuration to use the local Pulp container registry on control plane hosts
+* Configuration to deploy pre-built OS images to overcloud hosts using Bifrost
 
 Local Pulp server
 -----------------
@@ -60,6 +61,10 @@ Pulp startup.
 If a proxy is required to access the Internet from the seed, ``pulp_proxy_url``
 may be used.
 
+Host images are not synchronised to the local Pulp server, since they should
+only be pulled to the seed node once. More information on host images can be
+found :ref:`here <host-images>`.
+
 StackHPC Ark
 ------------
 
@@ -74,6 +79,9 @@ The Ark pulp credentials issued by StackHPC should be configured in
 Package repositories
 --------------------
 
+Rocky Linux 9 and Ubuntu Jammy package repositories are synced based on the
+value of ``os_distribution`` and ``os_release``.
+
 On Ark, each package repository provides versioned snapshots using a datetime
 stamp (e.g. ``20220817T082321``). The current set of tested versions is defined
 in ``etc/kayobe/pulp-repo-versions.yml``. This file is managed by the StackHPC
@@ -83,15 +91,28 @@ repository.
 Package managers
 ----------------
 
-For Rocky Linux based systems, package manager configuration is
-provided by ``stackhpc_dnf_repos`` in ``etc/kayobe/dnf.yml``, which points to
-package repositories on the local Pulp server. To use this configuration, the
+For Ubuntu Jammy systems, the package manager configuration is provided by
+``stackhpc_apt_repositories`` in ``etc/kayobe/apt.yml``.
+
+The configuration is applied by default to all Ubuntu hosts. The configuration
+can be overridden by changing the repository definitions in
+``apt_repositories`` or toggling ``apt_disable_sources_list`` to use the
+default apt repositories. This can be done on a host-by host basis by defining
+the variables as host or group vars under ``etc/kayobe/inventory/host_vars`` or
+``etc/kayobe/inventory/group_vars``.
+
+For Rocky Linux based systems, package manager configuration is provided by
+``stackhpc_dnf_repos`` in ``etc/kayobe/dnf.yml``, which points to package
+repositories on the local Pulp server. To use this configuration, the
 ``dnf_custom_repos`` variable must be set, and this is done for hosts in the
 ``overcloud`` group via the group_vars file
 ``etc/kayobe/inventory/group_vars/overcloud/stackhpc-dnf-repos``. Similar
 configuration may be added for other groups, however there may be ordering
 issues during initial deployment when Pulp has not yet been deployed.
 
+In both instances, the configuration points to package repositories on the
+local Pulp server.
+
 The distribution name for the environment should be configured as either
 ``development`` or ``production`` via ``stackhpc_repo_distribution`` in
 ``etc/kayobe/stackhpc.yml``.
@@ -102,6 +123,13 @@ Ceph container images
 By default, Ceph images are not synced from quay.io to the local Pulp. To sync
 these images, set ``stackhpc_sync_ceph_images`` to ``true``.
 
+HashiCorp container images
+--------------------------
+
+By default, HashiCorp images (Consul and Vault) are not synced from Docker Hub
+to the local Pulp. To sync these images, set ``stackhpc_sync_hashicorp_images``
+to ``true``.
+
 Usage
 =====
 
@@ -136,7 +164,7 @@ See the Kayobe :kayobe-doc:`custom playbook documentation
   these are new container image repositories, then the new image tags will not
   be available to cloud nodes until they have been published.
 * ``pulp-container-publish.yml``: Publish synchronised container images in the
-  local Pulp. This will make synchonised container images available to cloud
+  local Pulp. This will make synchronised container images available to cloud
   nodes.
 
 Syncing content
@@ -254,4 +282,4 @@ you will see a 404 error during ``pulp-repo-sync.yml``:
       msg: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/rocky/9/BaseOS/x86_64/os/20211122T102435')) '''
 
 The issue can be rectified by updating the ``stackhpc_release_pulp_username``
-and ``stackhpc_release_pulp_password`` variables
+and ``stackhpc_release_pulp_password`` variables.

doc/source/configuration/vault.rst

@@ -39,6 +39,11 @@ Before beginning the deployment of vault for openstack internal TLS and backend
   * Ansible Galaxy dependencies installed: ``kayobe control host bootstrap``
   * Python dependencies installed: ``pip install -r kayobe-config/requirements.txt``
 
+By default, Consul and Vault images are not synced from Docker Hub to the local
+Pulp. To sync these images, set ``stackhpc_sync_hashicorp_images`` to ``true``.
+The Vault deployment configuration will be automatically updated to pull images
+from Pulp.
+
 Deployment
 ==========
 
@@ -210,6 +215,16 @@ Enable the required TLS variables in kayobe and kolla
 
 3. Deploy backend and internal TLS
 
+   .. warning::
+
+      It is important that you are only using admin endpoints for keystone. If
+      any admin endpoints exist for other services, they must be deleted e.g.
+
+      .. code-block::
+
+         openstack endpoint list --interface admin -f value | \
+         awk '!/keystone/ {print $1}' | xargs openstack endpoint delete
+
    .. code-block::
 
       kayobe overcloud service deploy

doc/source/contributor/environments/aufn-ceph.rst

@@ -6,6 +6,11 @@ This environment creates a Universe-from-nothing_-style deployment of Kayobe con
 
 .. _Universe-from-nothing: https://github.com/stackhpc/a-universe-from-nothing
 
+.. warning::
+
+    This guide was written for the Yoga release and has not been validated for
+    Zed. Proceed with caution.
+
 Prerequisites
 =============