Merge pull request #20 from stackhpc/2024.12.1-sync

2024.12.1 sync

Author: MoteHue

.github/actions/setup/action.yml

@@ -107,6 +107,14 @@ runs:
         source ./bin/activate "$AZIMUTH_CONFIG_ENVIRONMENT" "$AZIMUTH_ENVIRONMENT"
         ansible-galaxy install -f -r requirements.yml
 
+    - name: Generate secrets for environment
+      shell: bash
+      run: |
+        set -e
+        source ci.env
+        source ./bin/activate "$AZIMUTH_CONFIG_ENVIRONMENT" "$AZIMUTH_ENVIRONMENT"
+        ./bin/generate-secrets
+
     # Generate and append the S3 credential to the CI environment file
     - name: Configure S3 lock
       id: s3-lock-config

.github/workflows/update-dependencies.yml

@@ -11,6 +11,7 @@ on:
 jobs:
   propose_github_release_updates:
     runs-on: ubuntu-latest
+    if: github.repository == 'azimuth-cloud/azimuth-config'
     strategy:
       matrix:
         include:

.gitignore

@@ -5,3 +5,6 @@
 .python-version
 /clouds.yaml*
 tilt-settings.yaml
+# Ignore generated secrets in demo and CI environments
+environments/demo/inventory/group_vars/all/secrets.yml
+.github/environments/**/secrets.yml

Tiltfile

@@ -8,6 +8,8 @@ TILT_IMAGES_UNAPPLY = os.path.abspath("./bin/tilt-images-unapply")
 # Allow the use of the azimuth-dev context
 allow_k8s_contexts("azimuth")
 
+# Increase the timeout for applying to Kubernetes
+update_settings(k8s_upsert_timeout_secs = 600)
 
 def deep_merge(dict1, dict2):
     """
@@ -34,6 +36,13 @@ if not os.path.exists(SETTINGS_FILE):
 # Load the settings
 settings = deep_merge(
     {
+        # The engine that will be used to build container images for your changes
+        # Supported options are docker, podman
+        "build_engine": "docker",
+        # The engine that will be used to mirror container images when required
+        # Supported options are skopeo (recommended), docker, podman
+        # Defaults to the build engine
+        # "mirror_engine": "skopeo",
         # The components that will be managed by Tilt, if locally available
         # By default, we search for local checkouts as siblings of this checkout
         "components": {
@@ -101,27 +110,23 @@ def build_image(name, context, build_args = None):
     """
     Defines an image build and returns the image name.
     """
+    build_engine = settings["build_engine"]
+    if build_engine not in ["docker", "podman"]:
+        fail("unknown build engine - %s" % build_engine)
     image = image_name(name)
-    # The Azimuth CaaS operator relies on the .git folder to be in the Docker build context
-    # This is because it uses pbr for versioning
-    # Unfortunately, Tilt's docker_build function _always_ ignores the .git directory :-(
+    # Some of the Azimuth components rely on the .git folder to be in the build context (pbr)
+    # Unfortunately, Tilt's {docker,podman}_build functions _always_ ignores the .git directory
     # So we use a custom build command
-    build_command = [
-        "docker",
-        "build",
-        "-t",
-        "$EXPECTED_REF",
-        "--platform",
-        "linux/amd64",
-        context,
-    ]
-    if build_args:
-        for arg_name, arg_value in build_args.items():
-            build_command.extend([
-                "--build-arg",
-                "'%s=%s'" % (arg_name, arg_value),
-            ])
-    custom_build(image, " ".join(build_command), [context])
+    build_args = " ".join([
+        item
+        for arg_name, arg_value in (build_args or {}).items()
+        for item in ["--build-arg", "'%s=%s'" % (arg_name, arg_value)]
+    ])
+    build_command = (
+        "%s build -t $EXPECTED_REF --platform linux/amd64 %s %s && " % (build_engine, build_args, context) +
+        "%s push $EXPECTED_REF" % build_engine
+    )
+    custom_build(image, build_command, [context], skips_local_docker = True)
     return image
 
 
@@ -130,14 +135,18 @@ def mirror_image(name, source_image):
     Defines a mirrored image and returns the image name.
     """
     image = image_name(name)
-    custom_build(
-        image,
-        (
-            "docker pull --platform linux/amd64 {source_image} && " +
-            "docker tag {source_image} $EXPECTED_REF"
-        ).format(source_image = source_image),
-        []
-    )
+    mirror_engine = settings.get("mirror_engine") or settings["build_engine"]
+    if mirror_engine in ["docker", "podman"]:
+        mirror_command = (
+            "%s pull --platform linux/amd64 %s && " % (mirror_engine, source_image) +
+            "%s tag %s $EXPECTED_REF && " % (mirror_engine, source_image) +
+            "%s push $EXPECTED_REF" % mirror_engine
+        )
+    elif mirror_engine == "skopeo":
+        mirror_command = "skopeo copy --all docker://%s docker://$EXPECTED_REF" % source_image
+    else:
+        fail("unrecognised mirror engine - %s" % mirror_engine)
+    custom_build(image, mirror_command, [], skips_local_docker = True)
     return image
 
 

bin/generate-secrets

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+#####
+## This script generates a secrets file for an environment.
+##
+## The environment can either be given as an argument or activated.
+#####
+
+set -eo pipefail
+
+
+# Parse the command line arguments
+# The environment defaults to the active environment, if set
+COMMAND_ENVIRONMENT="${AZIMUTH_CONFIG_ENVIRONMENT:-""}"
+FORCE_OVERWRITE=
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -f|--force)
+            FORCE_OVERWRITE="yes"
+            shift
+            ;;
+        *)
+            COMMAND_ENVIRONMENT="$1"
+            shift
+            ;;
+    esac
+done
+
+# If the environment is unknown at this point, bail
+if [ -z "$COMMAND_ENVIRONMENT" ]; then
+    echo "Target environment must either be specified as an argument or activated" >&2
+    exit 1
+fi
+
+# Work out where the secrets file for the specified environment lives
+CONFIG_ROOT="$(dirname $(dirname $(realpath ${BASH_SOURCE[0]:-${(%):-%x}})))"
+# We check environments and .github/environments, as in activate
+if [ -d "$CONFIG_ROOT/environments/$COMMAND_ENVIRONMENT" ]; then
+  CONFIG_ENVIRONMENT_ROOT="$CONFIG_ROOT/environments/$COMMAND_ENVIRONMENT"
+elif [ -d "$CONFIG_ROOT/.github/environments/$COMMAND_ENVIRONMENT" ]; then
+  CONFIG_ENVIRONMENT_ROOT="$CONFIG_ROOT/.github/environments/$COMMAND_ENVIRONMENT"
+else
+  echo "Unrecognised config environment '$COMMAND_ENVIRONMENT'" >&2
+  exit 1
+fi
+SECRETS_FILE="$CONFIG_ENVIRONMENT_ROOT/inventory/group_vars/all/secrets.yml"
+echo "Writing secrets to $SECRETS_FILE"
+
+# If the secrets file already exists, do not overwrite it unless explicitly requested
+if [ -f "$SECRETS_FILE" ]; then
+    if [ "$FORCE_OVERWRITE" = "yes" ]; then
+        echo "$SECRETS_FILE already exists - overwriting"
+    else
+        echo "$SECRETS_FILE already exists - will not overwrite" >&2
+        exit 1
+    fi
+fi
+
+# Write the secrets file, making sure the directory exists first
+mkdir -p "$(dirname $SECRETS_FILE)"
+cat <<EOF > $SECRETS_FILE
+#####
+# This file contains secrets for the $COMMAND_ENVIRONMENT environment
+#
+# It should be encrypted if stored in version control
+# https://azimuth-config.readthedocs.io/en/stable/repository/secrets/
+#####
+
+# https://azimuth-config.readthedocs.io/en/stable/configuration/05-secret-key/
+# The secret key for signing Azimuth cookies
+azimuth_secret_key: "$(openssl rand -hex 32)"
+
+# https://azimuth-config.readthedocs.io/en/stable/configuration/07-platform-identity/#keycloak-admin-password
+# The admin password for the Keycloak master realm
+keycloak_admin_password: "$(openssl rand -hex 16)"
+
+# https://azimuth-config.readthedocs.io/en/stable/configuration/08-zenith/
+# The secret key for signing Zenith registrar tokens
+zenith_registrar_subdomain_token_signing_key: "$(openssl rand -hex 32)"
+
+# https://azimuth-config.readthedocs.io/en/stable/configuration/10-kubernetes-clusters/#harbor-registry
+# The password for the Harbor admin account
+harbor_admin_password: "$(openssl rand -hex 16)"
+# The secret key for Harbor
+harbor_secret_key: "$(openssl rand -hex 8)"
+
+# https://azimuth-config.readthedocs.io/en/stable/configuration/14-monitoring/#accessing-web-interfaces
+# The admin password for Azimuth administrative dashboards
+admin_dashboard_ingress_basic_auth_password: "$(openssl rand -hex 16)"
+EOF

bin/kube-connect

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
 #####
-## This script uses Tilt (tilt.dev) to allow easier code development on the
-## currently activated environment
+## This script allows access to the Azimuth Kubernetes cluster from the machine
+## where the script is executed by using a SOCKS proxy
 #####
 
 set -eo pipefail
@@ -80,7 +80,7 @@ if [ "$install_mode" = "ha" ]; then
     cluster_name="$(ansible_variable capi_cluster_release_name)"
     kubeconfig_arg="KUBECONFIG=./kubeconfig-${cluster_name}.yaml"
 fi
-"$AZIMUTH_CONFIG_ROOT/bin/seed-ssh" \
+"$AZIMUTH_CONFIG_ROOT/bin/seed-ssh" -q \
   $kubeconfig_arg \
   kubectl config view --raw > "$kubeconfig"
 
@@ -104,7 +104,7 @@ kubectl config rename-context $ctx azimuth --kubeconfig $kubeconfig >/dev/null
 
 # Launch the SOCKS proxy and store the PID
 echo "Starting SOCKS proxy..." >&2
-"$AZIMUTH_CONFIG_ROOT/bin/seed-ssh" -D $socks_port -N &
+"$AZIMUTH_CONFIG_ROOT/bin/seed-ssh" -q -D $socks_port -N &
 socks_pid="$!"
 # Wait a few seconds and check that the process is running
 sleep 5

bin/seed-ssh

@@ -13,7 +13,6 @@ if [ -z "$AZIMUTH_CONFIG_ROOT" ] || [ -z "$AZIMUTH_CONFIG_ENVIRONMENT_ROOT" ]; t
     exit 1
 fi
 
-
 ansible_variable() {
     ANSIBLE_LOAD_CALLBACK_PLUGINS=true \
     ANSIBLE_STDOUT_CALLBACK=json \
@@ -22,6 +21,15 @@ ansible_variable() {
        jq -r -R "fromjson? | .plays[0].tasks[0].hosts.localhost.$1"
 }
 
+tf_init() {
+    ansible_variable terraform_backend_config > "$terraform_dir/backend_config.json"
+    $terraform_binary_path \
+      -chdir="$terraform_dir" \
+      init \
+      -input=false \
+      -reconfigure \
+      -backend-config=$terraform_dir/backend_config.json
+}
 
 # Add the Terraform binary directory to the PATH, so we can use it if it was
 # downloaded as part of a provision
@@ -45,6 +53,20 @@ fi
 work_dir="$(ansible_variable work_directory)/seed-ssh"
 mkdir -p "$work_dir"
 
+# Check if quiet mode (-q) was passed to SSH command
+# so that we can suppress other output elsewhere too
+QUIET_MODE=false
+for arg in $@; do
+  if [[ ! $arg == -* ]]; then
+    # Break if we encounter a non-flag arg since
+    # this is likely a command to run within the SSH
+    # session instead of an arg intended for SSH client
+    break
+  elif [[ $arg == "-q" ]]; then
+    QUIET_MODE=true
+  fi
+done
+
 # Initialise the OpenTofu backend
 terraform_backend_type="$(ansible_variable terraform_backend_type)"
 if [ "$terraform_backend_type" = "local" ]; then
@@ -60,13 +82,12 @@ terraform {
   backend "${terraform_backend_type}" {}
 }
 EOF
-    ansible_variable terraform_backend_config > "$terraform_dir/backend_config.json"
-    $terraform_binary_path \
-      -chdir="$terraform_dir" \
-      init \
-      -input=false \
-      -reconfigure \
-      -backend-config=$terraform_dir/backend_config.json
+  # If -q (quiet) is passed to ssh then also suppress terraform / tofu output
+  if [[ $QUIET_MODE == "true" ]]; then
+    tf_init > /dev/null
+  else
+    tf_init
+  fi
 fi
 
 # Read the required variables from the Terraform state

docs/configuration/03-kubernetes-config.md

@@ -130,8 +130,9 @@ etcd is extremely sensitive to write latency.
 
 Azimuth is able to configure Kubernetes nodes, both for the HA cluster and tenant clusters, so
 that etcd is on a separate block device. This block device can be of a different volume type to
-the root disk, allowing efficient use of SSD-backed storage. When supported by the flavor, the
-etcd block device can also use local disk even if the root volume is from Cinder.
+the root disk, allowing efficient use of SSD-backed storage. When the flavor has an ephemeral storage
+allocation, and the ephemeral storage capacity is at least as large as the desired etcd block device
+size, the etcd block device can also use local disk even if the root volume is from Cinder.
 
 !!! tip  "Use local disk for etcd whenever possible"