From b5b3b393f002adee47f9caaebc154a50f1297528 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 27 Aug 2025 13:18:00 +0000 Subject: [PATCH 1/5] remove dnf_repos extra index/key and make epel/openhpc special-cases simpler --- ansible/roles/dnf_repos/defaults/main.yml | 6 +- .../roles/dnf_repos/tasks/disable_repos.yml | 2 +- ansible/roles/dnf_repos/tasks/set_repos.yml | 8 +- .../inventory/group_vars/all/dnf_repos.yml | 28 +-- .../inventory/group_vars/all/timestamps.yml | 231 +++++++++--------- 5 files changed, 127 insertions(+), 148 deletions(-) diff --git a/ansible/roles/dnf_repos/defaults/main.yml b/ansible/roles/dnf_repos/defaults/main.yml index aa9757e57..fe3c44e4f 100644 --- a/ansible/roles/dnf_repos/defaults/main.yml +++ b/ansible/roles/dnf_repos/defaults/main.yml @@ -1,8 +1,4 @@ -dnf_repos_default: {} # see environments/common/inventory/group_vars/all/timestamps.yml -dnf_repos_extra: {} -dnf_repos_no_epel: {} -dnf_repos_all: {} -dnf_repos_default_epel: {} +dnf_repos_repos: {} # see environments/common/inventory/group_vars/all/{dnf_repos,timestamps}.yml dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content" dnf_repos_username: "{{ omit }}" dnf_repos_password: "{{ omit }}" diff --git a/ansible/roles/dnf_repos/tasks/disable_repos.yml b/ansible/roles/dnf_repos/tasks/disable_repos.yml index 61459e547..4db073bc1 100644 --- a/ansible/roles/dnf_repos/tasks/disable_repos.yml +++ b/ansible/roles/dnf_repos/tasks/disable_repos.yml @@ -7,7 +7,7 @@ description: "{{ repo_name }}" enabled: false gpgcheck: false - loop: "{{ dnf_repos_all | dict2items }}" + loop: "{{ dnf_repos_repos | dict2items }}" loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" vars: diff --git a/ansible/roles/dnf_repos/tasks/set_repos.yml b/ansible/roles/dnf_repos/tasks/set_repos.yml index f5d19207f..2db4de91b 100644 --- a/ansible/roles/dnf_repos/tasks/set_repos.yml +++ b/ansible/roles/dnf_repos/tasks/set_repos.yml @@ -1,6 +1,6 @@ --- -- name: Replace system repos with Pulp repos +- name: Replace non-epel repos with Pulp repos ansible.builtin.yum_repository: file: "{{ repo_values.repo_file }}" name: "{{ repo_name }}" @@ -9,9 +9,10 @@ username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" gpgcheck: false - loop: "{{ dnf_repos_no_epel | dict2items }}" + loop: "{{ dnf_repos_repos | dict2items }}" loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" + when: repo_name != 'epel' vars: repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}" repo_values: "{{ item.value[repo_os] }}" @@ -32,9 +33,10 @@ username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" gpgcheck: false - loop: "{{ dnf_repos_default_epel | dict2items }}" + loop: "{{ dnf_repos_repos | dict2items }}" loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" + when: repo_name == 'epel' vars: repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}" repo_values: "{{ item.value[repo_os] }}" diff --git a/environments/common/inventory/group_vars/all/dnf_repos.yml b/environments/common/inventory/group_vars/all/dnf_repos.yml index bd1029872..d920ed0de 100644 --- a/environments/common/inventory/group_vars/all/dnf_repos.yml +++ b/environments/common/inventory/group_vars/all/dnf_repos.yml @@ -1,24 +1,8 @@ -dnf_repos_all: "{{ dnf_repos_no_epel | combine(dnf_repos_default_epel) }}" -dnf_repos_no_epel: | - {{ dnf_repos_default['base'] - | combine(dnf_repos_default['ohpc'] if (openhpc_install_type | default('ohpc')) == 'ohpc' else {}) - | combine(dnf_repos_extra) }} -dnf_repos_default_epel: "{{ dnf_repos_default['epel'] }}" +# dnf_repos_default: see timestamps.yml -# see timestamps.yml for dnf_repos_default definition, default repos should be in format -# dnf_repos_default: -# base: # top level keys for internal indexing only, see `dnf_repos_all` and `dnf_repos_no_epel` -# appstream: # yum_repository:name -# '8.10': # ansible_distribution_version or ansible_distribution_major_version -# repo_file: Rocky-AppStream # yum_repository: file -# # repo_name: # optional, override yum_repository:name -# pulp_path: rocky/8.10/AppStream/x86_64/os -# pulp_timestamp: 20250614T013846 -# # pulp_content_url: # optional, dnf_repos_pulp_content_url +# override this in environments/site/inventory/group_vars/dnf_repos.yml to add repos: +dnf_repos_extra: {} -# Should be in same format as dnf_repos_default, except without the top level indexing keys e.g -# dnf_repos_extra: -# appstream: -# 8.10: -# ... -dnf_repos_extra: {} \ No newline at end of file +# indirection to skip openhpc repos if using alternative slurm: +dnf_repos_skip: "{{ [] if ((openhpc_install_type | default('ohpc') == 'ohpc')) else ['OpenHPC', 'OpenHPC-updates'] }}" +dnf_repos_repos: "{{ dnf_repos_default | combine(dnf_repos_extra) | dict2items | rejectattr('key', 'in', dnf_repos_skip) | items2dict }}" diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml index c8ff64c51..d2df041b4 100644 --- a/environments/common/inventory/group_vars/all/timestamps.yml +++ b/environments/common/inventory/group_vars/all/timestamps.yml @@ -1,119 +1,116 @@ dnf_repos_default: - base: - Ceph: - '8': - pulp_path: centos/8-stream/storage/x86_64/ceph-quincy - pulp_timestamp: 20231104T015751 - repo_file: ceph - '9': - pulp_path: centos/9-stream/storage/x86_64/ceph-reef - pulp_timestamp: 20250617T023108 - repo_file: ceph - appstream: - '8.10': - pulp_path: rocky/8.10/AppStream/x86_64/os - pulp_timestamp: 20250614T013846 - repo_file: Rocky-AppStream - '9.4': - pulp_path: rocky/9.4/AppStream/x86_64/os - pulp_timestamp: 20241112T003151 - repo_file: rocky - '9.5': - pulp_path: rocky/9.5/AppStream/x86_64/os - pulp_timestamp: 20250514T014704 - repo_file: rocky - '9.6': - pulp_path: rocky/9.6/AppStream/x86_64/os - pulp_timestamp: 20250816T020215 - repo_file: rocky - baseos: - '8.10': - pulp_path: rocky/8.10/BaseOS/x86_64/os - pulp_timestamp: 20250614T013846 - repo_file: Rocky-BaseOS - '9.4': - pulp_path: rocky/9.4/BaseOS/x86_64/os - pulp_timestamp: 20241115T011711 - repo_file: rocky - '9.5': - pulp_path: rocky/9.5/BaseOS/x86_64/os - pulp_timestamp: 20250513T031844 - repo_file: rocky - '9.6': - pulp_path: rocky/9.6/BaseOS/x86_64/os - pulp_timestamp: 20250815T050653 - repo_file: rocky - crb: - '8.10': - pulp_path: rocky/8.10/PowerTools/x86_64/os - pulp_timestamp: 20250614T013846 - repo_file: Rocky-PowerTools - repo_name: powertools - '9.4': - pulp_path: rocky/9.4/CRB/x86_64/os - pulp_timestamp: 20241115T003133 - repo_file: rocky - '9.5': - pulp_path: rocky/9.5/CRB/x86_64/os - pulp_timestamp: 20250514T014704 - repo_file: rocky - '9.6': - pulp_path: rocky/9.6/CRB/x86_64/os - pulp_timestamp: 20250815T034418 - repo_file: rocky - extras: - '8.10': - pulp_path: rocky/8.10/extras/x86_64/os - pulp_timestamp: 20250510T032327 - repo_file: Rocky-Extras - '9.4': - pulp_path: rocky/9.4/extras/x86_64/os - pulp_timestamp: 20241118T002802 - repo_file: rocky-extras - '9.5': - pulp_path: rocky/9.5/extras/x86_64/os - pulp_timestamp: 20250506T032818 - repo_file: rocky-extras - '9.6': - pulp_path: rocky/9.6/extras/x86_64/os - pulp_timestamp: 20250726T040613 - repo_file: rocky-extras - grafana: - '8': - pulp_path: grafana/oss/rpm - pulp_timestamp: 20250730T011314 - repo_file: grafana - timestamp: 20250615T005738 - '9': - pulp_path: grafana/oss/rpm - pulp_timestamp: 20250730T011314 - repo_file: grafana + Ceph: + '8': + pulp_path: centos/8-stream/storage/x86_64/ceph-quincy + pulp_timestamp: 20231104T015751 + repo_file: ceph + '9': + pulp_path: centos/9-stream/storage/x86_64/ceph-reef + pulp_timestamp: 20250617T023108 + repo_file: ceph + appstream: + '8.10': + pulp_path: rocky/8.10/AppStream/x86_64/os + pulp_timestamp: 20250614T013846 + repo_file: Rocky-AppStream + '9.4': + pulp_path: rocky/9.4/AppStream/x86_64/os + pulp_timestamp: 20241112T003151 + repo_file: rocky + '9.5': + pulp_path: rocky/9.5/AppStream/x86_64/os + pulp_timestamp: 20250514T014704 + repo_file: rocky + '9.6': + pulp_path: rocky/9.6/AppStream/x86_64/os + pulp_timestamp: 20250816T020215 + repo_file: rocky + baseos: + '8.10': + pulp_path: rocky/8.10/BaseOS/x86_64/os + pulp_timestamp: 20250614T013846 + repo_file: Rocky-BaseOS + '9.4': + pulp_path: rocky/9.4/BaseOS/x86_64/os + pulp_timestamp: 20241115T011711 + repo_file: rocky + '9.5': + pulp_path: rocky/9.5/BaseOS/x86_64/os + pulp_timestamp: 20250513T031844 + repo_file: rocky + '9.6': + pulp_path: rocky/9.6/BaseOS/x86_64/os + pulp_timestamp: 20250815T050653 + repo_file: rocky + crb: + '8.10': + pulp_path: rocky/8.10/PowerTools/x86_64/os + pulp_timestamp: 20250614T013846 + repo_file: Rocky-PowerTools + repo_name: powertools + '9.4': + pulp_path: rocky/9.4/CRB/x86_64/os + pulp_timestamp: 20241115T003133 + repo_file: rocky + '9.5': + pulp_path: rocky/9.5/CRB/x86_64/os + pulp_timestamp: 20250514T014704 + repo_file: rocky + '9.6': + pulp_path: rocky/9.6/CRB/x86_64/os + pulp_timestamp: 20250815T034418 + repo_file: rocky + extras: + '8.10': + pulp_path: rocky/8.10/extras/x86_64/os + pulp_timestamp: 20250510T032327 + repo_file: Rocky-Extras + '9.4': + pulp_path: rocky/9.4/extras/x86_64/os + pulp_timestamp: 20241118T002802 + repo_file: rocky-extras + '9.5': + pulp_path: rocky/9.5/extras/x86_64/os + pulp_timestamp: 20250506T032818 + repo_file: rocky-extras + '9.6': + pulp_path: rocky/9.6/extras/x86_64/os + pulp_timestamp: 20250726T040613 + repo_file: rocky-extras + grafana: + '8': + pulp_path: grafana/oss/rpm + pulp_timestamp: 20250730T011314 + repo_file: grafana + timestamp: 20250615T005738 + '9': + pulp_path: grafana/oss/rpm + pulp_timestamp: 20250730T011314 + repo_file: grafana epel: - epel: - '8': - pulp_path: epel/8/Everything/x86_64 - pulp_timestamp: 20250615T234151 - repo_file: epel - '9': - pulp_path: epel/9/Everything/x86_64 - pulp_timestamp: 20250817T000753 - repo_file: epel - ohpc: - OpenHPC: - '8': - pulp_path: OpenHPC/2/EL_8 - pulp_timestamp: 20241218T154614 - repo_file: OpenHPC - '9': - pulp_path: OpenHPC/3/EL_9 - pulp_timestamp: 20241218T154614 - repo_file: OpenHPC - OpenHPC-updates: - '8': - pulp_path: OpenHPC/2/updates/EL_8 - pulp_timestamp: 20250512T003315 - repo_file: OpenHPC - '9': - pulp_path: OpenHPC/3/updates/EL_9 - pulp_timestamp: 20250510T003301 - repo_file: OpenHPC + '8': + pulp_path: epel/8/Everything/x86_64 + pulp_timestamp: 20250615T234151 + repo_file: epel + '9': + pulp_path: epel/9/Everything/x86_64 + pulp_timestamp: 20250817T000753 + repo_file: epel + OpenHPC: + '8': + pulp_path: OpenHPC/2/EL_8 + pulp_timestamp: 20241218T154614 + repo_file: OpenHPC + '9': + pulp_path: OpenHPC/3/EL_9 + pulp_timestamp: 20241218T154614 + repo_file: OpenHPC + OpenHPC-updates: + '8': + pulp_path: OpenHPC/2/updates/EL_8 + pulp_timestamp: 20250512T003315 + repo_file: OpenHPC + '9': + pulp_path: OpenHPC/3/updates/EL_9 + pulp_timestamp: 20250510T003301 + repo_file: OpenHPC From 244a9ee5688f0670df5ccf50f20d64525d095780 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 27 Aug 2025 13:22:59 +0000 Subject: [PATCH 2/5] clarify pulp distro selection --- ansible/adhoc/sync-pulp.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/adhoc/sync-pulp.yml b/ansible/adhoc/sync-pulp.yml index c879d438b..373f3abb8 100644 --- a/ansible/adhoc/sync-pulp.yml +++ b/ansible/adhoc/sync-pulp.yml @@ -6,4 +6,5 @@ vars: pulp_site_target_arch: "x86_64" pulp_site_target_distribution: "rocky" - pulp_site_target_distribution_version: "{{ dnf_repos_all['baseos'].keys() | map('float') | sort | last }}" \ No newline at end of file + # default distribution to *latest* specified for baseos repo: + pulp_site_target_distribution_version: "{{ dnf_repos_repos['baseos'].keys() | map('float') | sort | last }}" From 32278e7e99a5fc7b7fb782828b6b70653ed83b65 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 27 Aug 2025 13:48:07 +0000 Subject: [PATCH 3/5] fixup sync vars --- ansible/roles/pulp_site/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml index 1eedf907c..3d2bce7ec 100644 --- a/ansible/roles/pulp_site/defaults/main.yml +++ b/ansible/roles/pulp_site/defaults/main.yml @@ -18,7 +18,7 @@ pulp_site_rpm_repo_defaults: _pulp_site_rpm_info: | {{ - dnf_repos_all | + dnf_repos_repos | select_repos(pulp_site_target_distribution_version) }} pulp_site_rpm_repos: | From a63fa1fd8f4aa7489d68fe7f93ea1131ecb04ce8 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 27 Aug 2025 13:48:25 +0000 Subject: [PATCH 4/5] fixup grafana vars --- environments/common/files/grafana/grafana.repo.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/common/files/grafana/grafana.repo.j2 b/environments/common/files/grafana/grafana.repo.j2 index cc736edf2..6ce2581f6 100644 --- a/environments/common/files/grafana/grafana.repo.j2 +++ b/environments/common/files/grafana/grafana.repo.j2 @@ -1,6 +1,6 @@ {{ ansible_managed | comment }} [grafana] -baseurl = {{ appliances_pulp_url }}/pulp/content/{{ dnf_repos_all['grafana'][ansible_distribution_major_version]['pulp_path'] }}/{{ dnf_repos_all['grafana'][ansible_distribution_major_version]['pulp_timestamp'] }} +baseurl = {{ appliances_pulp_url }}/pulp/content/{{ dnf_repos_repos['grafana'][ansible_distribution_major_version]['pulp_path'] }}/{{ dnf_repos_repos['grafana'][ansible_distribution_major_version]['pulp_timestamp'] }} enabled = 0 name = grafana async = 1 From 4580c2ec9a884022119cd28af6869c7cffd2d80a Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 27 Aug 2025 14:02:00 +0000 Subject: [PATCH 5/5] revert latest timestamp changes for extra key level --- ansible/library/latest_timestamps.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/ansible/library/latest_timestamps.py b/ansible/library/latest_timestamps.py index fc82ab8cf..0de388326 100644 --- a/ansible/library/latest_timestamps.py +++ b/ansible/library/latest_timestamps.py @@ -52,21 +52,17 @@ def run_module(): ) timestamps = dict(module.params['repos_dict']) - for index in timestamps: - for repo in timestamps[index]: - for version in timestamps[index][repo]: + for repo in timestamps: + for version in timestamps[repo]: - html_txt = requests.get( - url= module.params['content_url'] + '/' + timestamps[index][repo][version]['pulp_path'] - ).text - timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html - timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags - latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed - timestamps[index][repo][version]['pulp_timestamp'] = latest_timestamp - # timestamps[index] = dict(sorted(timestamps[index])) - - - result['timestamps'] = timestamps + html_txt = requests.get( + url= module.params['content_url'] + '/' + timestamps[repo][version]['pulp_path'] + ).text + timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html + timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags + latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed + timestamps[repo][version]['pulp_timestamp'] = latest_timestamp + result['timestamps'] = dict(sorted(timestamps.items())) module.exit_json(**result)