wip: get pulp sync working

Author: sjpb

ansible/adhoc/sync-pulp.yml

@@ -6,5 +6,4 @@
       vars:
         pulp_site_target_arch: "x86_64"
         pulp_site_target_distribution: "rocky"
-        pulp_site_target_distribution_version: "9.5"
-        pulp_site_target_distribution_version_major: "9"
+        pulp_site_target_distribution_version: "9.5" # TODO: how can we automatically make this the latest??

ansible/roles/pulp_site/defaults/main.yml

@@ -8,37 +8,25 @@ pulp_site_validate_certs: false
 pulp_site_install_dir: '/home/rocky/pulp'
 pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}"
 pulp_site_target_facts: "{{ hostvars[groups['pulp'][0]]['ansible_facts'] }}"
-pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}"
-pulp_site_target_distribution_version_major: "{{ pulp_site_target_facts['distribution_major_version'] }}"
-
-pulp_site_rpm_info:
-- name: "baseos-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.baseos[pulp_site_target_distribution_version].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.baseos[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
-- name: "appstream-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.appstream[pulp_site_target_distribution_version].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.appstream[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
-- name: "crb-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.crb[pulp_site_target_distribution_version].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.crb[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
-- name: "extras-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.extras[pulp_site_target_distribution_version].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.extras[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
-- name: "epel-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.epel[pulp_site_target_distribution_version_major].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.epel[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
-- name: "ohpc-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.openhpc_base[pulp_site_target_distribution_version_major].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.openhpc_base[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
-- name: "ohpc-updates-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
-- name: "ceph-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
-- name: "grafana-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.grafana[pulp_site_target_distribution_version_major].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.grafana[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
+pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}" # TODO: how to set automatically?
 
 pulp_site_rpm_repo_defaults:
   remote_username: "{{ pulp_site_upstream_username }}"
   remote_password: "{{ pulp_site_upstream_password }}"
   policy: on_demand
   state: present
 
-_pulp_site_rpm_info_all: "{{ pulp_site_rpm_info | map('combine', pulp_site_rpm_repo_defaults) }}"
+pulp_site_rpm_info: |
+  {{
+    dnf_repos_default |
+    combine(dnf_repos_default_epel, dnf_repos_extra) |
+    select_repos(pulp_site_target_distribution_version)
+  }}
+pulp_site_rpm_repos: |
+  {{
+    pulp_site_rpm_info |
+    to_rpm_repos(pulp_site_upstream_content_url, pulp_site_rpm_repo_defaults)
+  }}
 
-pulp_site_rpm_repos: "{{ _pulp_site_rpm_info_all | to_rpm_repos(pulp_site_upstream_content_url) }}"
 pulp_site_rpm_publications: "{{ _pulp_site_rpm_info_all | to_rpm_pubs }}"
 pulp_site_rpm_distributions: "{{ _pulp_site_rpm_info_all | to_rpm_distros }}"

ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py

@@ -3,29 +3,51 @@ def filters(self):
         return {
             'to_rpm_repos': self.to_rpm_repos,
             'to_rpm_pubs': self.to_rpm_pubs,
-            'to_rpm_distros': self.to_rpm_distros
+            'to_rpm_distros': self.to_rpm_distros,
+            'select_repos': self.select_repos,
         }
-
-    def to_rpm_repos(self, list, pulp_url):
-        repo_list = map(lambda x: {
-            'name': x['name'],
-            'url': pulp_url+'/'+x['subpath'],
-            'remote_username': x['remote_username'],
-            'remote_password': x['remote_password'],
-            'policy': x['policy'],
-            'state': x['state'] }, list)
-        return repo_list
 
+    def select_repos(self, dnf_repos, target_distro_ver):
+        """ Filter dnf_repos to only those for a relevant distribution version (M.m or M). Returns a list of dicts.
+            TODO: note this adds distro_ver as a key
+        """
+    
+        target_distro_ver_major = target_distro_ver.split('.')[0]
+
+        rpm_repos = []
+        for repokey in dnf_repos:
+            # select either the matching major.minor or major version:
+            if target_distro_ver in dnf_repos[repokey]:
+                selected_ver = target_distro_ver
+            elif target_distro_ver_major in dnf_repos[repokey]:
+                selected_ver = target_distro_ver_major
+            else:
+                raise ValueError(f'No key matching {target_distro_ver_major} or {target_distro_ver} found in f{repokey}')
+            repo_data = dnf_repos[repokey][selected_ver]
+            repo_data['distro_ver'] = selected_ver
+            rpm_repos.append(repo_data)
+        return rpm_repos
+
+    def to_rpm_repos(self, rpm_info, content_url, repo_defaults):
+        """ TODO """
+        rpm_repos = []
+        for repo_data in rpm_info:
+            rpm_data = repo_defaults.copy() # NB: this changes behaviour vs before, so now defaults can correctly be overriden
+            rpm_data['name'] = f"{repo_data['pulp_repo_name']}-{repo_data['distro_ver']}-{repo_data['pulp_timestamp']}"
+            rpm_data['url'] = '/'.join([content_url, repo_data['pulp_path'], repo_data['pulp_timestamp']])
+            rpm_repos.append(rpm_data)
+        return rpm_repos
+
     def to_rpm_pubs(self, list):
         pub_list = map(lambda x: {
-            'repository': x['name'],
+            'repository': x['pulp_repo_name'],
             'state': x['state'] }, list)
         return pub_list
 
     def to_rpm_distros(self, list):
         distro_list = map(lambda x: {
             'name': x['name'],
-            'repository': x['name'],
-            'base_path': x['subpath'],
+            'repository': x['pulp_repo_name'],
+            'base_path': x['pulp_path'],
             'state': x['state'] }, list)
-        return distro_list
+        return distro_list

docs/experimental/pulp.md

@@ -7,6 +7,8 @@ In order to ensure reproducible builds, the appliance can build images using rep
 ### Deploying a Pulp server
 A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. To use this, create an inventory file defining a group `pulp_server` containing a single host. The hostvar `ansible_host` should be defined, giving the IP address Ansible should use for ssh.
 
+**TODO: should be RL9 (or RL8?)**
+**TODO: add size required**
 **TODO: example inventory file**
 
 Once complete, it will print a message giving a value to set for `appliances_pulp_url`, assuming the `ansible_host` address is also the address the cluster
@@ -17,6 +19,8 @@ should use to reach the Pulp server.
 Note access to this server's content isn't authenticated so this assumes the `pulp_server` host is not externall reachable.
 
 **TODO: You can actually do this using additional_nodes now, how would we make the pulp store persistant?**
+**TODO: don't advise that, we want single server for all environments**
+**TODO: Add a systemd unit to run pulp!**
 
 ### Using an existing Pulp server
 An existing Pulp server can be used to host Ark repos by overriding `pulp_site_password` and `appliances_pulp_url` in the target environment. Note that this assumes the same configuration as the appliance deployed Pulp i.e no content authentication.

environments/common/inventory/group_vars/all/timestamps.yml

@@ -6,116 +6,143 @@ dnf_repos_default:
       pulp_path: rocky/8.10/AppStream/x86_64/os
       pulp_timestamp: 20250614T013846
       # pulp_content_url:             # optional, dnf_repos_pulp_content_url
+      pulp_repo_name: appstream       # pulp repository name
     '9.4':
       repo_file: rocky
       pulp_path: rocky/9.4/AppStream/x86_64/os
       pulp_timestamp: 20241112T003151
+      pulp_repo_name: appstream
     '9.5':
       repo_file: rocky
       pulp_path: rocky/9.5/AppStream/x86_64/os
       pulp_timestamp: 20250514T014704
+      pulp_repo_name: appstream
     '9.6':
       repo_file: rocky
       pulp_path: rocky/9.6/AppStream/x86_64/os
       pulp_timestamp: 20250726T040613
+      pulp_repo_name: appstream
   baseos:
     '8.10':
       repo_file: Rocky-BaseOS
       pulp_path: rocky/8.10/BaseOS/x86_64/os
       pulp_timestamp: 20250614T013846
+      pulp_repo_name: baseos
     '9.4':
       repo_file: rocky
       pulp_path: rocky/9.4/BaseOS/x86_64/os
       pulp_timestamp: 20241115T011711
+      pulp_repo_name: baseos
     '9.5':
       repo_file: rocky
       pulp_path: rocky/9.5/BaseOS/x86_64/os
       pulp_timestamp: 20250513T031844
+      pulp_repo_name: baseos
     '9.6':
       repo_file: rocky
       pulp_path: rocky/9.6/BaseOS/x86_64/os
       pulp_timestamp: 20250726T052250
+      pulp_repo_name: baseos
   Ceph:
     '8':
       repo_file: ceph
       pulp_path: centos/8-stream/storage/x86_64/ceph-quincy
       pulp_timestamp: 20231104T015751
+      pulp_repo_name: baseos
     '9':
       repo_file: ceph
       pulp_path: centos/9-stream/storage/x86_64/ceph-reef
       pulp_timestamp: 20250617T023108
+      pulp_repo_name: baseos
   crb:
     '8.10':
       repo_name: powertools
       repo_file: Rocky-PowerTools
       pulp_path: rocky/8.10/PowerTools/x86_64/os
       pulp_timestamp: 20250614T013846
+      pulp_repo_name: crb
     '9.4':
       repo_file: rocky
       pulp_path: rocky/9.4/CRB/x86_64/os
       pulp_timestamp: 20241115T003133
+      pulp_repo_name: crb
     '9.5':
       repo_file: rocky
       pulp_path: rocky/9.5/CRB/x86_64/os
       pulp_timestamp: 20250514T014704
+      pulp_repo_name: crb
     '9.6':
       repo_file: rocky
       pulp_path: rocky/9.6/CRB/x86_64/os
       pulp_timestamp: 20250726T040613
+      pulp_repo_name: crb
   extras:
     '8.10':
       repo_file: Rocky-Extras
       pulp_path: rocky/8.10/extras/x86_64/os
       pulp_timestamp: 20250510T032327
+      pulp_repo_name: extras
     '9.4':
       repo_file: rocky-extras
       pulp_path: rocky/9.4/extras/x86_64/os
       pulp_timestamp: 20241118T002802
+      pulp_repo_name: extras
     '9.5':
       repo_file: rocky-extras
       pulp_path: rocky/9.5/extras/x86_64/os
       pulp_timestamp: 20250506T032818
+      pulp_repo_name: extras
     '9.6':
       repo_file: rocky-extras
       pulp_path: rocky/9.6/extras/x86_64/os
       pulp_timestamp: 20250726T040613
+      pulp_repo_name: extras
   grafana:
     '8':
       repo_file: grafana
       pulp_path: grafana/oss/rpm
       timestamp: 20250615T005738
+      pulp_repo_name: grafana
     '9':
       repo_file: grafana
       pulp_path: grafana/oss/rpm
       pulp_timestamp: 20250730T011314
+      pulp_repo_name: grafana
   # TODO: figure out how to break out openhpc repos out separately to allow for no-ohpc case
   openhpc_base:
     '8':
       repo_file: OpenHPC
       pulp_path: OpenHPC/2/EL_8
       pulp_timestamp: 20241218T154614
+      pulp_repo_name: ohpc
     '9':
       repo_file: OpenHPC
       pulp_path: OpenHPC/3/EL_9
       pulp_timestamp: 20241218T154614
+      pulp_repo_name: ohpc
   openhpc_updates:
     '8':
       repo_file: OpenHPC
       pulp_path: OpenHPC/2/updates/EL_8
       pulp_timestamp: 20250512T003315
+      pulp_repo_name: ohpc-updates
     '9':
       repo_file: OpenHPC
       pulp_path: OpenHPC/3/updates/EL_9
       pulp_timestamp: 20250510T003301
+      pulp_repo_name: ohpc-updates
 
 dnf_repos_default_epel: # as for dnf_repos_default
   epel:
     '8':
       repo_file: epel
       pulp_path: epel/8/Everything/x86_64
       pulp_timestamp: 20250615T234151
+      pulp_repo_name: epel
     '9':
       repo_file: epel
       pulp_path: epel/9/Everything/x86_64
       pulp_timestamp: 20250729T235750
+      pulp_repo_name: epel
 
+dnf_repos_extra: {}