diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index 54777bd6cb..145cc3170f 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -212,7 +212,7 @@ jobs: source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe seed host command run \ - --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv" --show-output + --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv containerd docker.io docker-buildx" --show-output env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index 396fa40a59..657852dc80 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -190,17 +190,6 @@ jobs: localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3 EOF - # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details. - # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the - # runner and dind containers. - - name: Deploy an authenticating package repository mirror proxy - run: | - source venvs/kayobe/bin/activate && - source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy - env: - KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - - name: Create build logs output directory run: mkdir image-build-logs @@ -222,6 +211,9 @@ jobs: args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" args="$args -e kolla_build_log_path=$GITHUB_WORKSPACE/image-build-logs/kolla-build-overcloud.log" args="$args -e base_path=$GITHUB_WORKSPACE/opt/kayobe" + # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the + # runner and dind containers. + args="$args -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy" source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe overcloud container image build $args diff --git a/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml b/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml new file mode 120000 index 0000000000..6e84a71044 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml @@ -0,0 +1 @@ +../../../../../ansible/pulp-auth-proxy.yml \ No newline at end of file diff --git a/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml b/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml new file mode 120000 index 0000000000..6e84a71044 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml @@ -0,0 +1 @@ +../../../../../ansible/pulp-auth-proxy.yml \ No newline at end of file diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index 18a9958ab5..981c641dd5 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -36,6 +36,7 @@ kolla_build_neutron_ovs: true # Host and port of a package repository mirror. # Build against the development Pulp service repositories. # Use Ark's package repositories to install packages. +stackhpc_repo_mirror_auth_proxy_enabled: true stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}" stackhpc_repo_mirror_username: "skc-ci-aio" stackhpc_repo_mirror_password: !vault | diff --git a/etc/kayobe/environments/ci-builder/stackhpc-overcloud-host-images.yml b/etc/kayobe/environments/ci-builder/stackhpc-overcloud-host-images.yml new file mode 100644 index 0000000000..886b1773a7 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/stackhpc-overcloud-host-images.yml @@ -0,0 +1,3 @@ +--- +# Whether or not to build overcloud host images from Ark +stackhpc_build_overcloud_image_from_pulp_package_mirrors: true diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index e824bdf5a0..625e86fa18 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -64,11 +64,7 @@ overcloud_dib_host_packages_extra: # List of additional git repositories containing Diskimage Builder (DIB) # elements. See stackhpc.openstack.os_images role for usage. Default is empty. -overcloud_dib_git_elements_extra: - - repo: "https://github.com/stackhpc/stackhpc-image-elements" - local: "{{ source_checkout_path }}/stackhpc-image-elements" - version: "v1.6.3" - elements_path: "elements" +overcloud_dib_git_elements_extra: "{{ stackhpc_dib_image_elements_repos }}" # List of git repositories containing Diskimage Builder (DIB) elements. See # stackhpc.openstack.os_images role for usage. Default is a combination of diff --git a/etc/kayobe/pulp-host-image-versions.yml b/etc/kayobe/pulp-host-image-versions.yml index 4a189c65b2..b118afb5b8 100644 --- a/etc/kayobe/pulp-host-image-versions.yml +++ b/etc/kayobe/pulp-host-image-versions.yml @@ -1,5 +1,5 @@ --- # Overcloud host image versioning tags # These images must be in SMS, since they are used by our AIO CI runners -stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20250620T125648" -stackhpc_ubuntu_noble_overcloud_host_image_version: "2025.1-20250620T125648" +stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20250916T145046" +stackhpc_ubuntu_noble_overcloud_host_image_version: "2025.1-20250916T145046" diff --git a/etc/kayobe/stackhpc-overcloud-dib.yml b/etc/kayobe/stackhpc-overcloud-dib.yml index 99f5ae8362..89d6ac5984 100644 --- a/etc/kayobe/stackhpc-overcloud-dib.yml +++ b/etc/kayobe/stackhpc-overcloud-dib.yml @@ -22,6 +22,7 @@ stackhpc_overcloud_dib_name: "deployment_image" # StackHPC overcloud DIB image elements. stackhpc_overcloud_dib_elements: - "{{ os_distribution }}-{% if os_distribution == 'rocky' %}container-stackhpc{% else %}minimal{% endif %}" + - "{% if os_distribution == 'ubuntu' and stackhpc_repo_mirror_auth_proxy_enabled %}apt-no-verify-peer{% endif %}" - "cloud-init-datasources" - "{% if os_distribution == 'rocky' %}selinux-permissive{% endif %}" - "enable-serial-console" @@ -34,7 +35,9 @@ stackhpc_overcloud_dib_elements: - "{% if os_distribution == 'ubuntu' %}sudoers{% endif %}" # StackHPC overcloud DIB image environment variables. -stackhpc_overcloud_dib_env_vars: +stackhpc_overcloud_dib_env_vars: "{{ stackhpc_overcloud_dib_env_vars_default | combine(stackhpc_overcloud_dib_env_vars_ark if stackhpc_build_overcloud_image_from_pulp_package_mirrors | bool else {}) }}" + +stackhpc_overcloud_dib_env_vars_default: DIB_BLOCK_DEVICE_CONFIG: "{{ stackhpc_overcloud_dib_block_device_config_uefi_lvm }}" DIB_BOOTLOADER_DEFAULT_CMDLINE: "nofb nomodeset gfxpayload=text net.ifnames=1 rd.auto" DIB_GRUB_TIMEOUT: "5" @@ -43,22 +46,22 @@ stackhpc_overcloud_dib_env_vars: DIB_CONTAINERFILE_RUNTIME: "docker" DIB_CONTAINERFILE_NETWORK_DRIVER: "host" DIB_CONTAINERFILE_DOCKERFILE: "/opt/kayobe/src/stackhpc-image-elements/elements/rocky-container-stackhpc/containerfiles/9-stackhpc" - # NOTE: Not currently syncing Ubuntu packages, since the on_demand mirror in - # Ark does not work if the upstream mirror pulls packages (which it does - # sometimes). - # DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}" DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG: "{{ stackhpc_overcloud_dib_dracut_enabled_modules_default_config }}" DIB_RELEASE: "{{ overcloud_dib_os_release }}" DIB_SUDOERS_FILENAME: "no-fqdn" # Avoid DNS queries during sudo commands, since we might not always have working DNS. DIB_SUDOERS_CONFIG: | Defaults !fqdn - # FIXME: Support templating repo files. - # DIB_YUM_MINIMAL_BOOTSTRAP_REPOS: /path/to/dir/containing/dib-mirror-*.repo YUM: dnf # Workaround for stack user home ownership bug DIB_IMAGE_CACHE: "/tmp/yum" +stackhpc_overcloud_dib_env_vars_ark: + DIB_CONTAINERFILE_BUILDOPTS: >- + --build-arg=ROCKY_USE_MIRRORS=true + --build-arg=ROCKY_MIRROR_URLS={{ [stackhpc_repo_rocky_9_baseos_url, stackhpc_repo_rocky_9_appstream_url] | join(',') }} + DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_noble_url if os_distribution == 'ubuntu' else '' }}" + # StackHPC overcloud DIB image packages. stackhpc_overcloud_dib_packages: - "ethtool" diff --git a/etc/kayobe/stackhpc-overcloud-host-images.yml b/etc/kayobe/stackhpc-overcloud-host-images.yml index c009f8e8fe..59c3626b8e 100644 --- a/etc/kayobe/stackhpc-overcloud-host-images.yml +++ b/etc/kayobe/stackhpc-overcloud-host-images.yml @@ -5,6 +5,9 @@ # Whether or not to download overcloud host images from Ark stackhpc_download_overcloud_host_images: false +# Whether or not to build overcloud host images from Ark +stackhpc_build_overcloud_image_from_pulp_package_mirrors: false + # The overcloud host image source, defined by os_distribution, os_release, # and the current stable version. stackhpc_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url_with_auth }}/kayobe-images/\ diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 4588d1e02f..46520315d9 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -201,3 +201,12 @@ stackhpc_amphora_image_version: "2025.1-20250619T113933" # Secret store to deploy as a Certificate Authority. # Valid options are "vault" and "openbao". Default is "openbao". stackhpc_ca_secret_store: openbao + +############################################################################### +# diskimage-builder elements repos + +stackhpc_dib_image_elements_repos: + - repo: "https://github.com/stackhpc/stackhpc-image-elements" + local: "{{ source_checkout_path }}/stackhpc-image-elements" + version: "v1.6.4" + elements_path: "elements" diff --git a/releasenotes/notes/build-images-from-ark-6eb0d5c8779bb970.yaml b/releasenotes/notes/build-images-from-ark-6eb0d5c8779bb970.yaml new file mode 100644 index 0000000000..d1486252ee --- /dev/null +++ b/releasenotes/notes/build-images-from-ark-6eb0d5c8779bb970.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Added support for building overcloud host images from Pulp package mirrors. + This feature can be toggled with the flag + ``stackhpc_build_overcloud_image_from_pulp_package_mirrors``. By default, + this feature is enabled in the ``ci-builder`` environment and disabled + otherwise. StackHPC overcloud host images will now be built with packages + from Ark.