Merge pull request #765 from stackhpc/dnf-repos-sjpb

wtripp180901 · web-flow · commit 4d412ecf2f96 · 2025-09-01T11:08:58.000+01:00
remove extra level of key indentation
diff --git a/ansible/adhoc/sync-pulp.yml b/ansible/adhoc/sync-pulp.yml
@@ -6,4 +6,5 @@
       vars:
         pulp_site_target_arch: "x86_64"
         pulp_site_target_distribution: "rocky"
-        pulp_site_target_distribution_version: "{{ dnf_repos_all['baseos'].keys() | map('float') | sort | last }}"
+        # default distribution to *latest* specified for baseos repo:
+        pulp_site_target_distribution_version: "{{ dnf_repos_repos['baseos'].keys() | map('float') | sort | last }}"
diff --git a/ansible/library/latest_timestamps.py b/ansible/library/latest_timestamps.py
@@ -52,21 +52,17 @@ def run_module():
     )
 
     timestamps = dict(module.params['repos_dict'])
-    for index in timestamps:
-        for repo in timestamps[index]:
-            for version in timestamps[index][repo]:
+    for repo in timestamps:
+        for version in timestamps[repo]:
 
-                html_txt = requests.get(
-                        url= module.params['content_url'] + '/' + timestamps[index][repo][version]['pulp_path']
-                    ).text
-                timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html
-                timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags
-                latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
-                timestamps[index][repo][version]['pulp_timestamp'] = latest_timestamp
-        # timestamps[index] = dict(sorted(timestamps[index]))
-            
-    
-    result['timestamps'] = timestamps
+            html_txt = requests.get(
+                    url= module.params['content_url'] + '/' + timestamps[repo][version]['pulp_path']
+                ).text
+            timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html
+            timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags
+            latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
+            timestamps[repo][version]['pulp_timestamp'] = latest_timestamp
+    result['timestamps'] = dict(sorted(timestamps.items()))
 
     module.exit_json(**result)
 
diff --git a/ansible/roles/dnf_repos/defaults/main.yml b/ansible/roles/dnf_repos/defaults/main.yml
@@ -1,8 +1,4 @@
-dnf_repos_default: {} # see environments/common/inventory/group_vars/all/timestamps.yml
-dnf_repos_extra: {}
-dnf_repos_no_epel: {}
-dnf_repos_all: {}
-dnf_repos_default_epel: {}
+dnf_repos_repos: {} # see environments/common/inventory/group_vars/all/{dnf_repos,timestamps}.yml
 dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content"
 dnf_repos_username: "{{ omit }}"
 dnf_repos_password: "{{ omit }}"
diff --git a/ansible/roles/dnf_repos/tasks/disable_repos.yml b/ansible/roles/dnf_repos/tasks/disable_repos.yml
@@ -7,7 +7,7 @@
     description: "{{ repo_name }}"
     enabled: false
     gpgcheck: false
-  loop: "{{ dnf_repos_all | dict2items }}"
+  loop: "{{ dnf_repos_repos | dict2items }}"
   loop_control:
     label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
   vars:
diff --git a/ansible/roles/dnf_repos/tasks/set_repos.yml b/ansible/roles/dnf_repos/tasks/set_repos.yml
@@ -1,6 +1,6 @@
 ---
 
-- name: Replace system repos with Pulp repos
+- name: Replace non-epel repos with Pulp repos
   ansible.builtin.yum_repository:
     file: "{{ repo_values.repo_file }}"
     name: "{{ repo_name }}"
@@ -9,9 +9,10 @@
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
     gpgcheck: false
-  loop: "{{ dnf_repos_no_epel | dict2items }}"
+  loop: "{{ dnf_repos_repos | dict2items }}"
   loop_control:
     label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
+  when: repo_name != 'epel'
   vars:
     repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}"
     repo_values: "{{ item.value[repo_os] }}"
@@ -32,9 +33,10 @@
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
     gpgcheck: false
-  loop: "{{ dnf_repos_default_epel | dict2items }}"
+  loop: "{{ dnf_repos_repos | dict2items }}"
   loop_control:
     label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
+  when: repo_name == 'epel'
   vars:
     repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}"
     repo_values: "{{ item.value[repo_os] }}"
diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml
@@ -18,7 +18,7 @@ pulp_site_rpm_repo_defaults:
 
 _pulp_site_rpm_info: |
   {{
-    dnf_repos_all |
+    dnf_repos_repos |
     select_repos(pulp_site_target_distribution_version)
   }}
 pulp_site_rpm_repos: |
diff --git a/environments/common/files/grafana/grafana.repo.j2 b/environments/common/files/grafana/grafana.repo.j2
@@ -1,6 +1,6 @@
 {{ ansible_managed | comment }}
 [grafana]
-baseurl = {{ appliances_pulp_url }}/pulp/content/{{ dnf_repos_all['grafana'][ansible_distribution_major_version]['pulp_path'] }}/{{ dnf_repos_all['grafana'][ansible_distribution_major_version]['pulp_timestamp'] }}
+baseurl = {{ appliances_pulp_url }}/pulp/content/{{ dnf_repos_repos['grafana'][ansible_distribution_major_version]['pulp_path'] }}/{{ dnf_repos_repos['grafana'][ansible_distribution_major_version]['pulp_timestamp'] }}
 enabled = 0
 name = grafana
 async = 1
diff --git a/environments/common/inventory/group_vars/all/dnf_repos.yml b/environments/common/inventory/group_vars/all/dnf_repos.yml
@@ -1,24 +1,8 @@
-dnf_repos_all: "{{ dnf_repos_no_epel | combine(dnf_repos_default_epel) }}"
-dnf_repos_no_epel: |
-  {{ dnf_repos_default['base'] 
-    | combine(dnf_repos_default['ohpc'] if (openhpc_install_type | default('ohpc')) == 'ohpc' else {}) 
-    | combine(dnf_repos_extra) }}
-dnf_repos_default_epel: "{{ dnf_repos_default['epel'] }}"
+# dnf_repos_default: see timestamps.yml
 
-# see timestamps.yml for dnf_repos_default definition, default repos should be in format
-# dnf_repos_default:
-#   base:                                 # top level keys for internal indexing only, see `dnf_repos_all` and `dnf_repos_no_epel`
-#     appstream:                          # yum_repository:name
-#       '8.10':                           # ansible_distribution_version or ansible_distribution_major_version
-#         repo_file: Rocky-AppStream      # yum_repository: file
-#         # repo_name:                    # optional, override yum_repository:name
-#         pulp_path: rocky/8.10/AppStream/x86_64/os
-#         pulp_timestamp: 20250614T013846
-#         # pulp_content_url:             # optional, dnf_repos_pulp_content_url
+# override this in environments/site/inventory/group_vars/dnf_repos.yml to add repos:
+dnf_repos_extra: {}
 
-# Should be in same format as dnf_repos_default, except without the top level indexing keys e.g
-# dnf_repos_extra:
-#   appstream:
-#     8.10:
-#       ...
-dnf_repos_extra: {}
+# indirection to skip openhpc repos if using alternative slurm:
+dnf_repos_skip: "{{ [] if ((openhpc_install_type | default('ohpc') == 'ohpc')) else ['OpenHPC', 'OpenHPC-updates'] }}"
+dnf_repos_repos: "{{ dnf_repos_default | combine(dnf_repos_extra) | dict2items | rejectattr('key', 'in', dnf_repos_skip) | items2dict }}"
diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml
@@ -1,119 +1,116 @@
 dnf_repos_default:
-  base:
-    Ceph:
-      '8':
-        pulp_path: centos/8-stream/storage/x86_64/ceph-quincy
-        pulp_timestamp: 20231104T015751
-        repo_file: ceph
-      '9':
-        pulp_path: centos/9-stream/storage/x86_64/ceph-reef
-        pulp_timestamp: 20250617T023108
-        repo_file: ceph
-    appstream:
-      '8.10':
-        pulp_path: rocky/8.10/AppStream/x86_64/os
-        pulp_timestamp: 20250614T013846
-        repo_file: Rocky-AppStream
-      '9.4':
-        pulp_path: rocky/9.4/AppStream/x86_64/os
-        pulp_timestamp: 20241112T003151
-        repo_file: rocky
-      '9.5':
-        pulp_path: rocky/9.5/AppStream/x86_64/os
-        pulp_timestamp: 20250514T014704
-        repo_file: rocky
-      '9.6':
-        pulp_path: rocky/9.6/AppStream/x86_64/os
-        pulp_timestamp: 20250816T020215
-        repo_file: rocky
-    baseos:
-      '8.10':
-        pulp_path: rocky/8.10/BaseOS/x86_64/os
-        pulp_timestamp: 20250614T013846
-        repo_file: Rocky-BaseOS
-      '9.4':
-        pulp_path: rocky/9.4/BaseOS/x86_64/os
-        pulp_timestamp: 20241115T011711
-        repo_file: rocky
-      '9.5':
-        pulp_path: rocky/9.5/BaseOS/x86_64/os
-        pulp_timestamp: 20250513T031844
-        repo_file: rocky
-      '9.6':
-        pulp_path: rocky/9.6/BaseOS/x86_64/os
-        pulp_timestamp: 20250815T050653
-        repo_file: rocky
-    crb:
-      '8.10':
-        pulp_path: rocky/8.10/PowerTools/x86_64/os
-        pulp_timestamp: 20250614T013846
-        repo_file: Rocky-PowerTools
-        repo_name: powertools
-      '9.4':
-        pulp_path: rocky/9.4/CRB/x86_64/os
-        pulp_timestamp: 20241115T003133
-        repo_file: rocky
-      '9.5':
-        pulp_path: rocky/9.5/CRB/x86_64/os
-        pulp_timestamp: 20250514T014704
-        repo_file: rocky
-      '9.6':
-        pulp_path: rocky/9.6/CRB/x86_64/os
-        pulp_timestamp: 20250815T034418
-        repo_file: rocky
-    extras:
-      '8.10':
-        pulp_path: rocky/8.10/extras/x86_64/os
-        pulp_timestamp: 20250510T032327
-        repo_file: Rocky-Extras
-      '9.4':
-        pulp_path: rocky/9.4/extras/x86_64/os
-        pulp_timestamp: 20241118T002802
-        repo_file: rocky-extras
-      '9.5':
-        pulp_path: rocky/9.5/extras/x86_64/os
-        pulp_timestamp: 20250506T032818
-        repo_file: rocky-extras
-      '9.6':
-        pulp_path: rocky/9.6/extras/x86_64/os
-        pulp_timestamp: 20250726T040613
-        repo_file: rocky-extras
-    grafana:
-      '8':
-        pulp_path: grafana/oss/rpm
-        pulp_timestamp: 20250730T011314
-        repo_file: grafana
-        timestamp: 20250615T005738
-      '9':
-        pulp_path: grafana/oss/rpm
-        pulp_timestamp: 20250730T011314
-        repo_file: grafana
+  Ceph:
+    '8':
+      pulp_path: centos/8-stream/storage/x86_64/ceph-quincy
+      pulp_timestamp: 20231104T015751
+      repo_file: ceph
+    '9':
+      pulp_path: centos/9-stream/storage/x86_64/ceph-reef
+      pulp_timestamp: 20250617T023108
+      repo_file: ceph
+  appstream:
+    '8.10':
+      pulp_path: rocky/8.10/AppStream/x86_64/os
+      pulp_timestamp: 20250614T013846
+      repo_file: Rocky-AppStream
+    '9.4':
+      pulp_path: rocky/9.4/AppStream/x86_64/os
+      pulp_timestamp: 20241112T003151
+      repo_file: rocky
+    '9.5':
+      pulp_path: rocky/9.5/AppStream/x86_64/os
+      pulp_timestamp: 20250514T014704
+      repo_file: rocky
+    '9.6':
+      pulp_path: rocky/9.6/AppStream/x86_64/os
+      pulp_timestamp: 20250816T020215
+      repo_file: rocky
+  baseos:
+    '8.10':
+      pulp_path: rocky/8.10/BaseOS/x86_64/os
+      pulp_timestamp: 20250614T013846
+      repo_file: Rocky-BaseOS
+    '9.4':
+      pulp_path: rocky/9.4/BaseOS/x86_64/os
+      pulp_timestamp: 20241115T011711
+      repo_file: rocky
+    '9.5':
+      pulp_path: rocky/9.5/BaseOS/x86_64/os
+      pulp_timestamp: 20250513T031844
+      repo_file: rocky
+    '9.6':
+      pulp_path: rocky/9.6/BaseOS/x86_64/os
+      pulp_timestamp: 20250815T050653
+      repo_file: rocky
+  crb:
+    '8.10':
+      pulp_path: rocky/8.10/PowerTools/x86_64/os
+      pulp_timestamp: 20250614T013846
+      repo_file: Rocky-PowerTools
+      repo_name: powertools
+    '9.4':
+      pulp_path: rocky/9.4/CRB/x86_64/os
+      pulp_timestamp: 20241115T003133
+      repo_file: rocky
+    '9.5':
+      pulp_path: rocky/9.5/CRB/x86_64/os
+      pulp_timestamp: 20250514T014704
+      repo_file: rocky
+    '9.6':
+      pulp_path: rocky/9.6/CRB/x86_64/os
+      pulp_timestamp: 20250815T034418
+      repo_file: rocky
+  extras:
+    '8.10':
+      pulp_path: rocky/8.10/extras/x86_64/os
+      pulp_timestamp: 20250510T032327
+      repo_file: Rocky-Extras
+    '9.4':
+      pulp_path: rocky/9.4/extras/x86_64/os
+      pulp_timestamp: 20241118T002802
+      repo_file: rocky-extras
+    '9.5':
+      pulp_path: rocky/9.5/extras/x86_64/os
+      pulp_timestamp: 20250506T032818
+      repo_file: rocky-extras
+    '9.6':
+      pulp_path: rocky/9.6/extras/x86_64/os
+      pulp_timestamp: 20250726T040613
+      repo_file: rocky-extras
+  grafana:
+    '8':
+      pulp_path: grafana/oss/rpm
+      pulp_timestamp: 20250730T011314
+      repo_file: grafana
+      timestamp: 20250615T005738
+    '9':
+      pulp_path: grafana/oss/rpm
+      pulp_timestamp: 20250730T011314
+      repo_file: grafana
   epel:
-    epel:
-      '8':
-        pulp_path: epel/8/Everything/x86_64
-        pulp_timestamp: 20250615T234151
-        repo_file: epel
-      '9':
-        pulp_path: epel/9/Everything/x86_64
-        pulp_timestamp: 20250817T000753
-        repo_file: epel
-  ohpc:
-    OpenHPC:
-      '8':
-        pulp_path: OpenHPC/2/EL_8
-        pulp_timestamp: 20241218T154614
-        repo_file: OpenHPC
-      '9':
-        pulp_path: OpenHPC/3/EL_9
-        pulp_timestamp: 20241218T154614
-        repo_file: OpenHPC
-    OpenHPC-updates:
-      '8':
-        pulp_path: OpenHPC/2/updates/EL_8
-        pulp_timestamp: 20250512T003315
-        repo_file: OpenHPC
-      '9':
-        pulp_path: OpenHPC/3/updates/EL_9
-        pulp_timestamp: 20250510T003301
-        repo_file: OpenHPC
+    '8':
+      pulp_path: epel/8/Everything/x86_64
+      pulp_timestamp: 20250615T234151
+      repo_file: epel
+    '9':
+      pulp_path: epel/9/Everything/x86_64
+      pulp_timestamp: 20250817T000753
+      repo_file: epel
+  OpenHPC:
+    '8':
+      pulp_path: OpenHPC/2/EL_8
+      pulp_timestamp: 20241218T154614
+      repo_file: OpenHPC
+    '9':
+      pulp_path: OpenHPC/3/EL_9
+      pulp_timestamp: 20241218T154614
+      repo_file: OpenHPC
+  OpenHPC-updates:
+    '8':
+      pulp_path: OpenHPC/2/updates/EL_8
+      pulp_timestamp: 20250512T003315
+      repo_file: OpenHPC
+    '9':
+      pulp_path: OpenHPC/3/updates/EL_9
+      pulp_timestamp: 20250510T003301
+      repo_file: OpenHPC
