updated docs, added gpg checks, simplified filters

Author: wtripp180901

ansible/adhoc/deploy-pulp.yml

@@ -2,7 +2,7 @@
 
 - name: Install pulp on server
   become: yes
-  hosts: pulp_server # TODO: add this to common/groups
+  hosts: pulp_server
   tasks:
   - name: Install pulp
     ansible.builtin.include_role:

ansible/roles/dnf_repos/tasks/disable_repos.yml

@@ -14,7 +14,7 @@
     file: epel
     description: "{{ dnf_repos_epel_description }}"
     baseurl: "{{ dnf_repos_epel_baseurl }}"
-    gpgcheck: false
+    gpgcheck: true
     enabled: false
 
 - name: Get all repo files

ansible/roles/dnf_repos/tasks/set_repos.yml

@@ -8,7 +8,7 @@
     description: "{{ repo_name }}"
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
-    gpgcheck: false
+    gpgcheck: true
   loop: "{{ dnf_repos_no_epel | dict2items }}"
   loop_control:
     label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
@@ -31,7 +31,7 @@
     description: "{{ repo_name }}"
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
-    gpgcheck: false # TODO: is this really false here and above??
+    gpgcheck: true
   loop: "{{ dnf_repos_default_epel | dict2items }}"
   loop_control:
     label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"

ansible/roles/pulp_site/defaults/main.yml

@@ -1,7 +1,10 @@
 pulp_site_url: "{{ appliances_pulp_url }}"
 pulp_site_port: 8080
 pulp_site_username: admin # shouldn't be changed
-pulp_site_password: "{{ vault_pulp_admin_password }}" #todo make more obvious this is different from the password needed for ark (pulp_site_upstream_password)
+pulp_site_password: "{{ vault_pulp_admin_password }}"
+# See environments/common/inventory/groups_vars/all/pulp.yml
+# pulp_site_upstream_username:
+# pulp_site_upstream_password:
 pulp_site_upstream_content_url: https://ark.stackhpc.com/pulp/content
 pulp_site_default_upstream_suffix: "{{ pulp_site_target_arch }}/os"
 pulp_site_validate_certs: false

ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py

@@ -9,7 +9,8 @@ def filters(self):
 
     def select_repos(self, dnf_repos, target_distro_ver): #TODO: why does baseos get a major and minor version?
         """ Filter dnf_repos to only those for a relevant distribution version (M.m or M). Returns a list of dicts.
-            TODO: note this adds distro_ver and pulp_repo_name as a key
+            Also adds pulp_repo_name field to give the repository a unique name in Pulp to be referenced by subsequent
+            filters
         """
 
         target_distro_ver_major = target_distro_ver.split('.')[0]
@@ -24,35 +25,39 @@ def select_repos(self, dnf_repos, target_distro_ver): #TODO: why does baseos get
             else:
                 raise ValueError(f'No key matching {target_distro_ver_major} or {target_distro_ver} found in f{repokey}')
             repo_data = dnf_repos[repokey][selected_ver]
-            repo_data['distro_ver'] = selected_ver
-            repo_data['pulp_repo_name'] = repokey
+            repo_data['pulp_repo_name'] = f"{repokey}-{selected_ver}-{dnf_repos[repokey][selected_ver]['pulp_timestamp']}"
             rpm_repos.append(repo_data)
         return rpm_repos
 
     def to_rpm_repos(self, rpm_info, content_url, repo_defaults):
-        """ TODO """
+        """ Filter repo object list given by select_repos into dict required by the pulp_repository_rpm_repos variable
+            from stackhpc.pulp.pulp_repository role
+        """
         rpm_repos = []
         for repo_data in rpm_info:
             rpm_data = repo_defaults.copy() # NB: this changes behaviour vs before, so now defaults can correctly be overriden
-            rpm_data['name'] = get_repo_name(repo_data)
+            rpm_data['name'] = repo_data['pulp_repo_name']
             rpm_data['url'] = '/'.join([content_url, repo_data['pulp_path'], repo_data['pulp_timestamp']])
             rpm_data['state'] = 'present'
             rpm_repos.append(rpm_data)
         return rpm_repos
 
     def to_rpm_pubs(self, list):
+        """ Filter repo object list given by select_repos into dict required by the pulp_publication_rpm variable
+            from stackhpc.pulp.pulp_publication role
+        """
         pub_list = map(lambda x: {
-            'repository': get_repo_name(x),
+            'repository': x['pulp_repo_name'],
             'state': 'present' }, list)
         return pub_list
 
     def to_rpm_distros(self, list):
+        """ Filter repo object list given by select_repos into dict required by the pulp_distirubtion_rpm variable
+            from stackhpc.pulp.pulp_distribution role
+        """
         distro_list = map(lambda x: {
             'name': x['pulp_repo_name'],
-            'repository': get_repo_name(x),
+            'repository': x['pulp_repo_name'],
             'base_path': '/'.join([x['pulp_path'],x['pulp_timestamp']]),
             'state': 'present' }, list)
         return distro_list
-
-def get_repo_name(dnf_repos_data):
-    return f"{dnf_repos_data['pulp_repo_name']}-{dnf_repos_data['distro_ver']}-{dnf_repos_data['pulp_timestamp']}"

docs/experimental/pulp.md

@@ -5,18 +5,17 @@ In order to ensure reproducible builds, the appliance can build images using rep
 ## Deploying/configuring Pulp Server
 
 ### Deploying a Pulp server
-A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. To use this, create an inventory file defining a group `pulp_server` containing a single host. The hostvar `ansible_host` should be defined, giving the IP address Ansible should use for ssh.
+A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. To use this, create an inventory file defining a group `pulp_server` containing a single host, which requires at least 2 vCPUs and 4GB RAM. Deploying and syncing Pulp has been tested on an RL9 host. The hostvar `ansible_host` should be defined, giving the IP address Ansible should use for ssh. For example:
 
-**TODO: should be RL9 (or RL8?)**
-**TODO: add size required (2 vCPUs, 4GB RAM)**
-**TODO: example inventory file**
+```
+[pulp_server]
+pulp_host ansible_host=<VM-ip-address> # Note the host name can't conflict with group names i.e can't be called `pulp` or `pulp_server`
+```
 
 Once complete, it will print a message giving a value to set for `appliances_pulp_url`, assuming the `ansible_host` address is also the address the cluster
 should use to reach the Pulp server.
 
-**TODO: example config**
-
-Note access to this server's content isn't authenticated so this assumes the `pulp_server` host is not externall reachable.
+Note access to this server's content isn't authenticated so this assumes the `pulp_server` host is not externally reachable.
 
 **TODO: You can actually do this using additional_nodes now, how would we make the pulp store persistant?**
 **TODO: don't advise that, we want single server for all environments**
@@ -29,4 +28,12 @@ An existing Pulp server can be used to host Ark repos by overriding `pulp_site_p
 
 If the `pulp` group is added to the Packer build groups, the local Pulp server will be synced with Ark on build. You must authenticate with Ark by overriding `pulp_site_upstream_username` and `pulp_site_upstream_password` with your vault encrypted Ark dev credentials. `dnf_repos_username` and `dnf_repos_password` must remain unset to access content from the local Pulp.
 
-Content can also be synced by running `ansible/adhoc/sync-pulp.yml`. By default this syncs repositories for Rocky 9.5 <TODO: is this correct?>  but this can be overridden by setting extra variables for `pulp_site_target_arch`, `pulp_site_target_distribution`, `pulp_site_target_distribution_version` and `pulp_site_target_distribution_version_major`.
+Content can also be synced by running `ansible/adhoc/sync-pulp.yml`. By default this syncs repositories for the latest version of Rocky supported by the appliance but this can be overridden by setting extra variables for `pulp_site_target_arch`, `pulp_site_target_distribution` and `pulp_site_target_distribution_version`.
+
+## Example config in site variables
+
+```
+appliances_pulp_url: "http://<pulp-host-ip>:8080"
+pulp_site_upstream_username: <Ark-username>
+pulp_site_upstream_password: <Ark-password>
+```

environments/common/files/grafana/grafana.repo.j2

@@ -1,6 +1,6 @@
 {{ ansible_managed | comment }}
 [grafana]
-baseurl = {{ appliances_pulp_url }}/pulp/content/{{ appliances_pulp_repos.grafana[ansible_distribution_major_version] | appliances_repo_to_subpath }}
+baseurl = {{ appliances_pulp_url }}/pulp/content/{{ dnf_repos_all['grafana'][ansible_distribution_version_major]['pulp_path'] }}/{{ dnf_repos_all['grafana'][ansible_distribution_major_version]['pulp_timestamp'] }}
 enabled = 0
 name = grafana
 async = 1

environments/common/inventory/group_vars/all/dnf_repos.yml

@@ -1,10 +1,11 @@
-dnf_repos_all: "{{ dnf_repos_no_epel | combine(dnf_repos_default_epel) }}" #see timestamps.yml
+dnf_repos_all: "{{ dnf_repos_no_epel | combine(dnf_repos_default_epel) }}"
 dnf_repos_no_epel: |
   {{ dnf_repos_default['base'] 
     | combine(dnf_repos_default['ohpc'] if (openhpc_install_type | default('ohpc')) == 'ohpc' else {}) 
     | combine(dnf_repos_extra) }}
 dnf_repos_default_epel: "{{ dnf_repos_default['epel'] }}"
-dnf_repos_extra: {}
+
+# see timestamps.yml for dnf_repos_default definition, default repos should be in format
 # dnf_repos_default:
 #   base:                                 # top level keys for internal indexing only, see `dnf_repos_all` and `dnf_repos_no_epel`
 #     appstream:                          # yum_repository:name
@@ -14,4 +15,11 @@ dnf_repos_extra: {}
 #         pulp_path: rocky/8.10/AppStream/x86_64/os
 #         pulp_timestamp: 20250614T013846
 #         # pulp_content_url:             # optional, dnf_repos_pulp_content_url
-#         pulp_repo_name: appstream       # pulp repository name
+#         pulp_repo_name: appstream       # pulp repository name
+
+# Should be in same format as dnf_repos_default, except without the top level indexing keys e.g
+# dnf_repos_extra:
+#   appstream:
+#     8.10:
+#       ...
+dnf_repos_extra: {}

environments/common/inventory/groups

@@ -213,4 +213,8 @@ extra_packages
 # Hosts to configure for node health checks - either entire 'compute' group or empty
 
 [pulp_server]
+# Host to deploy a Pulp server on and sync with mirrors of upstream Ark repositories. Should be a group containing a single VM provisioned
+# separately from the appliance. e.g
+# pulp_host ansible_host=<VM-ip-address>
+# Note the host name can't conflict with group names i.e can't be called `pulp` or `pulp_server`
 

environments/site/inventory/groups

@@ -157,3 +157,9 @@ compute
 # Should be set to `compute` if enabled
 # Note that this feature currently assumes all compute nodes are VMs, enabling
 # when the cluster contains baremetal compute nodes may lead to unexpected scheduling behaviour
+
+[pulp_server]
+# Host to deploy a Pulp server on and sync with mirrors of upstream Ark repositories. Should be a group containing a single VM provisioned
+# separately from the appliance. e.g
+# pulp_host ansible_host=<VM-ip-address>
+# Note the host name can't conflict with group names i.e can't be called `pulp` or `pulp_server`