Merge branch 'feat/pulp-builds' into feat/extra-packages

Author: wtripp180901

ansible/adhoc/deploy-pulp.yml

@@ -11,7 +11,6 @@
   become: yes
   hosts: _pulp_host
   tasks:
-
   - name: Install pulp
     ansible.builtin.include_role:
       name: pulp_site

ansible/roles/dnf_repos/defaults/main.yml

@@ -8,16 +8,16 @@ dnf_repos_password: "{{ omit }}"
 dnf_repos_repolist:
 - file: rocky
   name: baseos
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/BaseOS/{{ ansible_architecture }}/os/{{ appliances_repo_minor_timestamps[ansible_distribution_version].baseos }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/BaseOS/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.baseos[ansible_distribution_version] }}"
 - file: rocky
   name: appstream
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/AppStream/{{ ansible_architecture }}/os/{{ appliances_repo_minor_timestamps[ansible_distribution_version].appstream }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/AppStream/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.appstream[ansible_distribution_version] }}"
 - file: rocky
   name: crb
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/CRB/{{ ansible_architecture }}/os/{{ appliances_repo_minor_timestamps[ansible_distribution_version].crb }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/CRB/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.crb[ansible_distribution_version] }}"
 - file: rocky-extras
   name: extras
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/extras/{{ ansible_architecture }}/os/{{ appliances_repo_minor_timestamps[ansible_distribution_version].extras }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/extras/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.extras[ansible_distribution_version] }}"
 
-dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/epel/{{ ansible_distribution_major_version }}/Everything/{{ ansible_architecture }}/{{ appliances_repo_major_timestamps[ansible_distribution_major_version].epel }}"
+dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/epel/{{ ansible_distribution_major_version }}/Everything/{{ ansible_architecture }}/{{ appliances_repo_timestamps.epel[ansible_distribution_major_version] }}"
 dnf_repos_epel_description: "epel"

ansible/roles/pulp_site/defaults/main.yml

@@ -1,12 +1,10 @@
 pulp_site_url: "{{ appliances_pulp_url }}"
 pulp_site_port: 8080
 pulp_site_username: admin # shouldn't be changed
+pulp_site_password: "{{ vault_pulp_admin_password }}"
 pulp_site_upstream_content_url: https://ark.stackhpc.com/pulp/content
-pulp_site_upstream_username: slurm-app-ci
-pulp_site_upstream_password: "{{ lookup('ansible.builtin.env', 'ARK_PASSWORD') }}"
 _pulp_site_rocky_prefix: "{{ pulp_site_target_distribution }}/{{ pulp_site_target_distribution_version }}"
 pulp_site_default_upstream_suffix: "{{ pulp_site_target_arch }}/os"
-pulp_site_password: "{{ vault_pulp_admin_password }}"
 pulp_site_validate_certs: false
 pulp_site_install_dir: '/home/rocky/pulp'
 pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}"
@@ -15,20 +13,18 @@ pulp_site_target_arch: "{{ pulp_site_target_facts['architecture'] }}"
 pulp_site_target_distribution: "{{ pulp_site_target_facts['distribution'] | lower }}"
 pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}"
 pulp_site_target_distribution_version_major: "{{ pulp_site_target_facts['distribution_major_version'] }}"
-pulp_site_version_timestamps: "{{ appliances_repo_minor_timestamps[pulp_site_target_distribution_version] }}"
-pulp_site_major_version_timestamps: "{{ appliances_repo_major_timestamps[pulp_site_target_distribution_version_major] }}"
 
 pulp_site_rpm_info:
-- name: "baseos-{{ pulp_site_target_distribution_version }}-{{ pulp_site_version_timestamps.baseos }}"
-  subpath: "{{ _pulp_site_rocky_prefix }}/BaseOS/{{ pulp_site_default_upstream_suffix }}/{{ pulp_site_version_timestamps.baseos }}"
-- name: "appstream-{{ pulp_site_target_distribution_version }}-{{ pulp_site_version_timestamps.appstream }}"
-  subpath: "{{ _pulp_site_rocky_prefix }}/AppStream/{{ pulp_site_default_upstream_suffix }}/{{ pulp_site_version_timestamps.appstream }}"
-- name: "crb-{{ pulp_site_target_distribution_version }}-{{ pulp_site_version_timestamps.crb }}"
-  subpath: "{{ _pulp_site_rocky_prefix }}/{{ 'PowerTools' if pulp_site_target_distribution_version_major == '8' else 'CRB' }}/{{ pulp_site_default_upstream_suffix }}/{{ pulp_site_version_timestamps.crb }}"
-- name: "extras-{{ pulp_site_target_distribution_version }}-{{ pulp_site_version_timestamps.extras }}"
-  subpath: "{{ _pulp_site_rocky_prefix }}/extras/{{ pulp_site_default_upstream_suffix }}/{{ pulp_site_version_timestamps.extras }}"
-- name: "epel-{{ pulp_site_target_distribution_version_major }}-{{ pulp_site_major_version_timestamps.epel }}"
-  subpath: "epel/{{ pulp_site_target_distribution_version_major }}/Everything/{{ pulp_site_target_arch }}/{{ pulp_site_major_version_timestamps.epel }}"
+- name: "baseos-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.baseos[pulp_site_target_distribution_version] }}"
+  subpath: "{{ _pulp_site_rocky_prefix }}/BaseOS/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.baseos[pulp_site_target_distribution_version] }}"
+- name: "appstream-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.appstream[pulp_site_target_distribution_version] }}"
+  subpath: "{{ _pulp_site_rocky_prefix }}/AppStream/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.appstream[pulp_site_target_distribution_version] }}"
+- name: "crb-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.crb[pulp_site_target_distribution_version] }}"
+  subpath: "{{ _pulp_site_rocky_prefix }}/{{ 'PowerTools' if pulp_site_target_distribution_version_major == '8' else 'CRB' }}/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.crb[pulp_site_target_distribution_version] }}"
+- name: "extras-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.extras[pulp_site_target_distribution_version] }}"
+  subpath: "{{ _pulp_site_rocky_prefix }}/extras/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.extras[pulp_site_target_distribution_version] }}"
+- name: "epel-{{ pulp_site_target_distribution_version_major }}-{{ appliances_repo_timestamps.epel[pulp_site_target_distribution_version_major] }}"
+  subpath: "epel/{{ pulp_site_target_distribution_version_major }}/Everything/{{ pulp_site_target_arch }}/{{ appliances_repo_timestamps.epel[pulp_site_target_distribution_version_major] }}"
 
 pulp_site_rpm_repo_defaults:
   remote_username: "{{ pulp_site_upstream_username }}"

docs/experimental/pulp.md

@@ -1,17 +1,17 @@
 # Pulp Server
 
-In order to ensure reproducible builds, the appliance can build images using repository mirrors from StackHPC's Ark Pulp server. The appliance will sync relevant repositories to local Pulp server which will be used for image builds. Using a local server can be enabled by adding `pulp` to the build groups and overriding `dnf_repos_repolist` to point at content hosted on the local server.
+In order to ensure reproducible builds, the appliance can build images using repository mirrors from StackHPC's "Ark" Pulp server. The appliance can sync relevant repositories to a local Pulp server which will then be used instead of Ark. Using a local Pulp can be enabled by adding `pulp` to the build groups and overriding `appliances_pulp_url` to point at the local Pulp's URL.
 
 ## Deploying/configuring Pulp Server
 
 ### Deploying a Pulp server
 A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml' playbook. This can be run with
-`ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<host_ip>"`
-This will print a Pulp endpoint which can be copied to your environments as appropriate. Ensure that the server is accessible on the specified port. Note that this server's content isn't authenticated so assumes the server is deployed behind a secure network.
+`ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<target_host>"`
+where `target_host` is any resolvable host. This will print a Pulp URL which can be copied to your environments as appropriate. Ensure that the server is accessible on the specified port. Note access to this server's content isn't authenticated so assumes the server is deployed behind a secure network.
 
 ### Using an existing Pulp server
 An existing Pulp server can be used to host Ark repos by overriding `pulp_site_password` and `appliances_pulp_url` in the target environment. Note that this assumes the same configuration as the appliance deployed pulp i.e no content authentication.
 
 ## Syncing Pulp content with Ark
 
-If the `pulp` group is added to the Packer build groups, the local Pulp server will be synced with Ark on build. You must supply your Ark credentials, either by overriding `pulp_site_upstream_password` or setting environment variable `ARK_PASSWORD`. Content can also be synced by running `ansible/adhoc/sync-pulp.yml`, optionally setting extravars for `pulp_site_target_arch`, `pulp_site_target_distribution`, `pulp_site_target_distribution_version` and `pulp_site_target_distribution_version`.
+If the `pulp` group is added to the Packer build groups, the local Pulp server will be synced with Ark on build. You must authenticate with Ark by overriding `pulp_site_upstream_username` and `pulp_site_upstream_password` with your vault encrypted Ark dev credentials. `dnf_repos_username` and `dnf_repos_password` must remain unset to access content from the local Pulp. Content can also be synced by running `ansible/adhoc/sync-pulp.yml`. By default this syncs repositories for Rocky 9.4 with x86_64 architecture, but can be overriden by setting extravars for `pulp_site_target_arch`, `pulp_site_target_distribution`, `pulp_site_target_distribution_version` and `pulp_site_target_distribution_version_major`.

docs/image-build.md

@@ -17,7 +17,8 @@ The fat images StackHPC builds and tests in CI are available from [GitHub releas
 To build either a site-specific fat image from scratch, or to extend an existing StackHPC fat image:
 
 1. Ensure the current OpenStack credentials have sufficient authorisation to upload images (this may or may not require the `member` role for an application credential, depending on your OpenStack configuration).
-2. Create a Packer [variable definition file](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/variables#assigning-values-to-input-variables) at e.g. `environments/<environment>/builder.pkrvars.hcl` containing at a minimum:
+2. The provided dev credentials for StackHPC's "Ark" Pulp server must be added to the target environments. This is done by overriding `dnf_repos_username` and `dnf_repos_password` with your vault encrypted credentials in `environments/<base_environment>/inventory/group_vars/all/pulp.yml`. See the [experimental docs](experimental/pulp.md) if you wish instead wish to use a local Pulp server.
+3. Create a Packer [variable definition file](https://developer.hashicorp.com/packer/docs/templates/hcl_templates/variables#assigning-values-to-input-variables) at e.g. `environments/<environment>/builder.pkrvars.hcl` containing at a minimum:
 
     ```hcl
     flavor = "general.v1.small"                           # VM flavor to use for builder VMs
@@ -35,9 +36,9 @@ To build either a site-specific fat image from scratch, or to extend an existing
       - `update,control,login,compute`: The resultant image has all packages in the source image updated, and then packages for all types of nodes in the cluster are added. When using a GenericCloud image for `source_image_name` this builds a site-specific fat image from scratch.
       - One or more specific groups which are not enabled in the appliance by default, e.g. `lustre`. When using a StackHPC fat image for `source_image_name` this extends the image with just this additional functionality.
 
-3. Activate the venv and the relevant environment.
+4. Activate the venv and the relevant environment.
 
-4. Build images using the relevant variable definition file, e.g.:
+5. Build images using the relevant variable definition file, e.g.:
 
         cd packer/
         PACKER_LOG=1 /usr/bin/packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
@@ -52,7 +53,7 @@ To build either a site-specific fat image from scratch, or to extend an existing
 
       then delete the failed volume, select cancelling the build when Packer queries, and then retry. This is [Openstack bug 1823445](https://bugs.launchpad.net/cinder/+bug/1823445).
 
-5. The built image will be automatically uploaded to OpenStack with a name prefixed `openhpc` and including a timestamp and a shortened git hash.
+6. The built image will be automatically uploaded to OpenStack with a name prefixed `openhpc` and including a timestamp and a shortened git hash.
 
 # Build Process
 

environments/.stackhpc/inventory/group_vars/builder.yml

@@ -10,5 +10,10 @@
 # appliances_pulp_url: "{{ pulp_server_config[lookup('env','CI_CLOUD')].url }}"
 # pulp_site_password: "{{ pulp_server_config[lookup('env','CI_CLOUD')].password }}"
 
+# Alternatively, configure to use ark directly:
 dnf_repos_username: slurm-app-ci
 dnf_repos_password: "{{ lookup('env','ARK_PASSWORD') }}"
+
+# Can be set regardless of approach above:
+pulp_site_upstream_username: slurm-app-ci
+pulp_site_upstream_password: "{{ lookup('ansible.builtin.env', 'ARK_PASSWORD') }}"

environments/common/inventory/group_vars/all/defaults.yml

@@ -103,13 +103,14 @@ appliances_extra_packages: "{{ appliances_default_extra_packages + appliances_ot
 
 ###################### ark repo timestamps ###################################################
 
-appliances_repo_minor_timestamps:
-  '9.4':
-    baseos: 20240816T002610
-    appstream: 20240816T002610
-    crb: 20240816T002610
-    extras: 20240816T002610
-
-appliances_repo_major_timestamps:
-  '9':
-    epel: 20240902T080424
+appliances_repo_timestamps:
+  baseos:
+    '9.4': 20240816T002610
+  appstream:
+    '9.4': 20240816T002610
+  crb:
+    '9.4': 20240816T002610
+  extras:
+    '9.4': 20240816T002610
+  epel:
+    '9': 20240902T080424

environments/common/inventory/group_vars/all/pulp.yml

@@ -1 +1,11 @@
 pulp_site_port: 8080
+
+# If using Ark directly (no local Pulp server), override the following with Ark creds
+
+# dnf_repos_username:
+# dnf_repos_password:
+
+# If instead using local Pulp server, override below with Ark creds
+
+# pulp_site_upstream_username:
+# pulp_site_upstream_password:

environments/common/inventory/groups

@@ -152,5 +152,6 @@ freeipa_client
 # Hosts to replace system repos with Pulp repos
 builder
 
-[pulp:children]
-# Hosts used to run Pulp API commands
+[pulp]
+# Add builder to this group to enable automatically syncing of pulp during image build
+# Warning: when using Ark directly rather than a local Pulp server, adding hosts other than `builder` risks leaking Ark creds