autodetect latest in adhoc script, refactored timestamps to allow gated ohpc repos, fixed pulp site

Author: wtripp180901

ansible/adhoc/sync-pulp.yml

@@ -6,4 +6,4 @@
       vars:
         pulp_site_target_arch: "x86_64"
         pulp_site_target_distribution: "rocky"
-        pulp_site_target_distribution_version: "9.6" # TODO: how can we automatically make this the latest??
+        pulp_site_target_distribution_version: "{{ dnf_repos_all['baseos'].keys() | map('float') | sort | last }}"

ansible/roles/dnf_repos/defaults/main.yml

@@ -1,5 +1,7 @@
 dnf_repos_default: {} # see environments/common/inventory/group_vars/all/timestamps.yml
 dnf_repos_extra: {}
+dnf_repos_no_epel: {}
+dnf_repos_all: {}
 dnf_repos_default_epel: {}
 dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content"
 dnf_repos_username: "{{ omit }}"

ansible/roles/dnf_repos/tasks/set_repos.yml

@@ -9,7 +9,7 @@
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
     gpgcheck: false
-  loop: "{{ dnf_repos_default | combine(dnf_repos_extra) | dict2items }}"
+  loop: "{{ dnf_repos_no_epel | dict2items }}"
   loop_control:
     label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
   vars:
@@ -40,5 +40,3 @@
     repo_values: "{{ item.value[repo_os] }}"
     repo_name: "{{ repo_values.repo_name | default(item.key) }}"
     repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}"
-
-- meta: end_here

ansible/roles/pulp_site/defaults/main.yml

@@ -14,12 +14,10 @@ pulp_site_rpm_repo_defaults:
   remote_username: "{{ pulp_site_upstream_username }}"
   remote_password: "{{ pulp_site_upstream_password }}"
   policy: on_demand
-  state: present
 
 pulp_site_rpm_info: |
   {{
-    dnf_repos_default |
-    combine(dnf_repos_default_epel, dnf_repos_extra) |
+    dnf_repos_all |
     select_repos(pulp_site_target_distribution_version)
   }}
 pulp_site_rpm_repos: |
@@ -28,5 +26,5 @@ pulp_site_rpm_repos: |
     to_rpm_repos(pulp_site_upstream_content_url, pulp_site_rpm_repo_defaults)
   }}
 
-pulp_site_rpm_publications: "{{ _pulp_site_rpm_info_all | to_rpm_pubs }}"
-pulp_site_rpm_distributions: "{{ _pulp_site_rpm_info_all | to_rpm_distros }}"
+pulp_site_rpm_publications: "{{ pulp_site_rpm_info | to_rpm_pubs }}"
+pulp_site_rpm_distributions: "{{ pulp_site_rpm_info | to_rpm_distros }}"

ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py

@@ -7,7 +7,7 @@ def filters(self):
             'select_repos': self.select_repos,
         }
 
-    def select_repos(self, dnf_repos, target_distro_ver):
+    def select_repos(self, dnf_repos, target_distro_ver): #TODO: why does baseos get a major and minor version?
         """ Filter dnf_repos to only those for a relevant distribution version (M.m or M). Returns a list of dicts.
             TODO: note this adds distro_ver as a key
         """
@@ -33,21 +33,25 @@ def to_rpm_repos(self, rpm_info, content_url, repo_defaults):
         rpm_repos = []
         for repo_data in rpm_info:
             rpm_data = repo_defaults.copy() # NB: this changes behaviour vs before, so now defaults can correctly be overriden
-            rpm_data['name'] = f"{repo_data['pulp_repo_name']}-{repo_data['distro_ver']}-{repo_data['pulp_timestamp']}"
+            rpm_data['name'] = get_repo_name(repo_data)
             rpm_data['url'] = '/'.join([content_url, repo_data['pulp_path'], repo_data['pulp_timestamp']])
+            rpm_data['state'] = 'present'
             rpm_repos.append(rpm_data)
         return rpm_repos
 
     def to_rpm_pubs(self, list):
         pub_list = map(lambda x: {
-            'repository': x['pulp_repo_name'],
-            'state': x['state'] }, list)
+            'repository': get_repo_name(x),
+            'state': 'present' }, list)
         return pub_list
 
     def to_rpm_distros(self, list):
         distro_list = map(lambda x: {
-            'name': x['name'],
-            'repository': x['pulp_repo_name'],
+            'name': x['pulp_repo_name'],
+            'repository': get_repo_name(x),
             'base_path': x['pulp_path'],
-            'state': x['state'] }, list)
+            'state': 'present' }, list)
         return distro_list
+
+def get_repo_name(dnf_repos_data):
+    return f"{dnf_repos_data['pulp_repo_name']}-{dnf_repos_data['distro_ver']}-{dnf_repos_data['pulp_timestamp']}"

environments/common/inventory/group_vars/all/timestamps.yml

@@ -1,148 +1,155 @@
-dnf_repos_default:
-  appstream:                          # yum_repository:name
-    '8.10':                           # ansible_distribution_version or ansible_distribution_major_version
-      repo_file: Rocky-AppStream      # yum_repository: file
-      # repo_name:                    # optional, override yum_repository:name
-      pulp_path: rocky/8.10/AppStream/x86_64/os
-      pulp_timestamp: 20250614T013846
-      # pulp_content_url:             # optional, dnf_repos_pulp_content_url
-      pulp_repo_name: appstream       # pulp repository name
-    '9.4':
-      repo_file: rocky
-      pulp_path: rocky/9.4/AppStream/x86_64/os
-      pulp_timestamp: 20241112T003151
-      pulp_repo_name: appstream
-    '9.5':
-      repo_file: rocky
-      pulp_path: rocky/9.5/AppStream/x86_64/os
-      pulp_timestamp: 20250514T014704
-      pulp_repo_name: appstream
-    '9.6':
-      repo_file: rocky
-      pulp_path: rocky/9.6/AppStream/x86_64/os
-      pulp_timestamp: 20250726T040613
-      pulp_repo_name: appstream
-  baseos:
-    '8.10':
-      repo_file: Rocky-BaseOS
-      pulp_path: rocky/8.10/BaseOS/x86_64/os
-      pulp_timestamp: 20250614T013846
-      pulp_repo_name: baseos
-    '9.4':
-      repo_file: rocky
-      pulp_path: rocky/9.4/BaseOS/x86_64/os
-      pulp_timestamp: 20241115T011711
-      pulp_repo_name: baseos
-    '9.5':
-      repo_file: rocky
-      pulp_path: rocky/9.5/BaseOS/x86_64/os
-      pulp_timestamp: 20250513T031844
-      pulp_repo_name: baseos
-    '9.6':
-      repo_file: rocky
-      pulp_path: rocky/9.6/BaseOS/x86_64/os
-      pulp_timestamp: 20250726T052250
-      pulp_repo_name: baseos
-  Ceph:
-    '8':
-      repo_file: ceph
-      pulp_path: centos/8-stream/storage/x86_64/ceph-quincy
-      pulp_timestamp: 20231104T015751
-      pulp_repo_name: baseos
-    '9':
-      repo_file: ceph
-      pulp_path: centos/9-stream/storage/x86_64/ceph-reef
-      pulp_timestamp: 20250617T023108
-      pulp_repo_name: baseos
-  crb:
-    '8.10':
-      repo_name: powertools
-      repo_file: Rocky-PowerTools
-      pulp_path: rocky/8.10/PowerTools/x86_64/os
-      pulp_timestamp: 20250614T013846
-      pulp_repo_name: crb
-    '9.4':
-      repo_file: rocky
-      pulp_path: rocky/9.4/CRB/x86_64/os
-      pulp_timestamp: 20241115T003133
-      pulp_repo_name: crb
-    '9.5':
-      repo_file: rocky
-      pulp_path: rocky/9.5/CRB/x86_64/os
-      pulp_timestamp: 20250514T014704
-      pulp_repo_name: crb
-    '9.6':
-      repo_file: rocky
-      pulp_path: rocky/9.6/CRB/x86_64/os
-      pulp_timestamp: 20250726T040613
-      pulp_repo_name: crb
-  extras:
-    '8.10':
-      repo_file: Rocky-Extras
-      pulp_path: rocky/8.10/extras/x86_64/os
-      pulp_timestamp: 20250510T032327
-      pulp_repo_name: extras
-    '9.4':
-      repo_file: rocky-extras
-      pulp_path: rocky/9.4/extras/x86_64/os
-      pulp_timestamp: 20241118T002802
-      pulp_repo_name: extras
-    '9.5':
-      repo_file: rocky-extras
-      pulp_path: rocky/9.5/extras/x86_64/os
-      pulp_timestamp: 20250506T032818
-      pulp_repo_name: extras
-    '9.6':
-      repo_file: rocky-extras
-      pulp_path: rocky/9.6/extras/x86_64/os
-      pulp_timestamp: 20250726T040613
-      pulp_repo_name: extras
-  grafana:
-    '8':
-      repo_file: grafana
-      pulp_path: grafana/oss/rpm
-      timestamp: 20250615T005738
-      pulp_repo_name: grafana
-    '9':
-      repo_file: grafana
-      pulp_path: grafana/oss/rpm
-      pulp_timestamp: 20250730T011314
-      pulp_repo_name: grafana
-  # TODO: figure out how to break out openhpc repos out separately to allow for no-ohpc case
-  openhpc_base:
-    '8':
-      repo_file: OpenHPC
-      pulp_path: OpenHPC/2/EL_8
-      pulp_timestamp: 20241218T154614
-      pulp_repo_name: ohpc
-    '9':
-      repo_file: OpenHPC
-      pulp_path: OpenHPC/3/EL_9
-      pulp_timestamp: 20241218T154614
-      pulp_repo_name: ohpc
-  openhpc_updates:
-    '8':
-      repo_file: OpenHPC
-      pulp_path: OpenHPC/2/updates/EL_8
-      pulp_timestamp: 20250512T003315
-      pulp_repo_name: ohpc-updates
-    '9':
-      repo_file: OpenHPC
-      pulp_path: OpenHPC/3/updates/EL_9
-      pulp_timestamp: 20250510T003301
-      pulp_repo_name: ohpc-updates
+dnf_repos_all: "{{ dnf_repos_no_epel | combine(dnf_repos_default_epel) }}"
+dnf_repos_no_epel: |
+  {{ dnf_repos_default['base'] 
+    | combine(dnf_repos_default['ohpc'] if (openhpc_install_type | default('ohpc')) == 'ohpc' else {}) 
+    | combine(dnf_repos_extra) }}
+dnf_repos_default_epel: "{{ dnf_repos_default['epel'] }}"
 
-dnf_repos_default_epel: # as for dnf_repos_default
+dnf_repos_default:
+  base:                                 # top level keys for internal indexing only, see ansible/roles/dnf_repos/defaults.yml
+    appstream:                          # yum_repository:name
+      '8.10':                           # ansible_distribution_version or ansible_distribution_major_version
+        repo_file: Rocky-AppStream      # yum_repository: file
+        # repo_name:                    # optional, override yum_repository:name
+        pulp_path: rocky/8.10/AppStream/x86_64/os
+        pulp_timestamp: 20250614T013846
+        # pulp_content_url:             # optional, dnf_repos_pulp_content_url
+        pulp_repo_name: appstream       # pulp repository name
+      '9.4':
+        repo_file: rocky
+        pulp_path: rocky/9.4/AppStream/x86_64/os
+        pulp_timestamp: 20241112T003151
+        pulp_repo_name: appstream
+      '9.5':
+        repo_file: rocky
+        pulp_path: rocky/9.5/AppStream/x86_64/os
+        pulp_timestamp: 20250514T014704
+        pulp_repo_name: appstream
+      '9.6':
+        repo_file: rocky
+        pulp_path: rocky/9.6/AppStream/x86_64/os
+        pulp_timestamp: 20250726T040613
+        pulp_repo_name: appstream
+    baseos:
+      '8.10':
+        repo_file: Rocky-BaseOS
+        pulp_path: rocky/8.10/BaseOS/x86_64/os
+        pulp_timestamp: 20250614T013846
+        pulp_repo_name: baseos
+      '9.4':
+        repo_file: rocky
+        pulp_path: rocky/9.4/BaseOS/x86_64/os
+        pulp_timestamp: 20241115T011711
+        pulp_repo_name: baseos
+      '9.5':
+        repo_file: rocky
+        pulp_path: rocky/9.5/BaseOS/x86_64/os
+        pulp_timestamp: 20250513T031844
+        pulp_repo_name: baseos
+      '9.6':
+        repo_file: rocky
+        pulp_path: rocky/9.6/BaseOS/x86_64/os
+        pulp_timestamp: 20250726T052250
+        pulp_repo_name: baseos
+    Ceph:
+      '8':
+        repo_file: ceph
+        pulp_path: centos/8-stream/storage/x86_64/ceph-quincy
+        pulp_timestamp: 20231104T015751
+        pulp_repo_name: ceph
+      '9':
+        repo_file: ceph
+        pulp_path: centos/9-stream/storage/x86_64/ceph-reef
+        pulp_timestamp: 20250617T023108
+        pulp_repo_name: ceph
+    crb:
+      '8.10':
+        repo_name: powertools
+        repo_file: Rocky-PowerTools
+        pulp_path: rocky/8.10/PowerTools/x86_64/os
+        pulp_timestamp: 20250614T013846
+        pulp_repo_name: crb
+      '9.4':
+        repo_file: rocky
+        pulp_path: rocky/9.4/CRB/x86_64/os
+        pulp_timestamp: 20241115T003133
+        pulp_repo_name: crb
+      '9.5':
+        repo_file: rocky
+        pulp_path: rocky/9.5/CRB/x86_64/os
+        pulp_timestamp: 20250514T014704
+        pulp_repo_name: crb
+      '9.6':
+        repo_file: rocky
+        pulp_path: rocky/9.6/CRB/x86_64/os
+        pulp_timestamp: 20250726T040613
+        pulp_repo_name: crb
+    extras:
+      '8.10':
+        repo_file: Rocky-Extras
+        pulp_path: rocky/8.10/extras/x86_64/os
+        pulp_timestamp: 20250510T032327
+        pulp_repo_name: extras
+      '9.4':
+        repo_file: rocky-extras
+        pulp_path: rocky/9.4/extras/x86_64/os
+        pulp_timestamp: 20241118T002802
+        pulp_repo_name: extras
+      '9.5':
+        repo_file: rocky-extras
+        pulp_path: rocky/9.5/extras/x86_64/os
+        pulp_timestamp: 20250506T032818
+        pulp_repo_name: extras
+      '9.6':
+        repo_file: rocky-extras
+        pulp_path: rocky/9.6/extras/x86_64/os
+        pulp_timestamp: 20250726T040613
+        pulp_repo_name: extras
+    grafana:
+      '8':
+        repo_file: grafana
+        pulp_path: grafana/oss/rpm
+        timestamp: 20250615T005738
+        pulp_repo_name: grafana
+      '9':
+        repo_file: grafana
+        pulp_path: grafana/oss/rpm
+        pulp_timestamp: 20250730T011314
+        pulp_repo_name: grafana
+  ohpc:
+    openhpc_base:
+      '8':
+        repo_file: OpenHPC
+        pulp_path: OpenHPC/2/EL_8
+        pulp_timestamp: 20241218T154614
+        pulp_repo_name: ohpc
+      '9':
+        repo_file: OpenHPC
+        pulp_path: OpenHPC/3/EL_9
+        pulp_timestamp: 20241218T154614
+        pulp_repo_name: ohpc
+    openhpc_updates:
+      '8':
+        repo_file: OpenHPC
+        pulp_path: OpenHPC/2/updates/EL_8
+        pulp_timestamp: 20250512T003315
+        pulp_repo_name: ohpc-updates
+      '9':
+        repo_file: OpenHPC
+        pulp_path: OpenHPC/3/updates/EL_9
+        pulp_timestamp: 20250510T003301
+        pulp_repo_name: ohpc-updates
   epel:
-    '8':
-      repo_file: epel
-      pulp_path: epel/8/Everything/x86_64
-      pulp_timestamp: 20250615T234151
-      pulp_repo_name: epel
-    '9':
-      repo_file: epel
-      pulp_path: epel/9/Everything/x86_64
-      pulp_timestamp: 20250729T235750
-      pulp_repo_name: epel
+    epel:
+      '8':
+        repo_file: epel
+        pulp_path: epel/8/Everything/x86_64
+        pulp_timestamp: 20250615T234151
+        pulp_repo_name: epel
+      '9':
+        repo_file: epel
+        pulp_path: epel/9/Everything/x86_64
+        pulp_timestamp: 20250729T235750
+        pulp_repo_name: epel
 
 dnf_repos_extra: {}