From 134b627d2be3b5eb39be9af02c6bc65b43424397 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Thu, 7 Aug 2025 14:53:43 +0000 Subject: [PATCH 01/30] WIP: refactor repos definitions --- ansible/filter_plugins/utils.py | 6 - ansible/roles/dnf_repos/defaults/main.yml | 2 + ansible/roles/dnf_repos/tasks/set_repos.yml | 19 +- .../inventory/group_vars/all/timestamps.yml | 173 +++++++++--------- 4 files changed, 106 insertions(+), 94 deletions(-) diff --git a/ansible/filter_plugins/utils.py b/ansible/filter_plugins/utils.py index b5b92ed7e..42b7107ee 100644 --- a/ansible/filter_plugins/utils.py +++ b/ansible/filter_plugins/utils.py @@ -61,11 +61,6 @@ def to_ood_regex(items): r = ['(%s)' % v for v in r] return '|'.join(r) -def appliances_repo_to_subpath(repo_entry): - """ Take an element from appliances_pulp_repos and convert it to a pulp path. This assumes that the remote and local pulp structures are the same - """ - return repo_entry['path'] + '/' + repo_entry['timestamp'] - class FilterModule(object): ''' Ansible core jinja2 filters ''' @@ -81,5 +76,4 @@ def filters(self): 'exists': exists, 'warn': self.warn, 'to_ood_regex': to_ood_regex, - 'appliances_repo_to_subpath': appliances_repo_to_subpath } diff --git a/ansible/roles/dnf_repos/defaults/main.yml b/ansible/roles/dnf_repos/defaults/main.yml index 9302eff84..cd19405fc 100644 --- a/ansible/roles/dnf_repos/defaults/main.yml +++ b/ansible/roles/dnf_repos/defaults/main.yml @@ -1,3 +1,5 @@ +dnf_repos_default: {} # see environments/common/inventory/group_vars/all/timestamps.yml +dnf_repos_extra: {} dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content" dnf_repos_username: "{{ omit }}" dnf_repos_password: "{{ omit }}" diff --git a/ansible/roles/dnf_repos/tasks/set_repos.yml b/ansible/roles/dnf_repos/tasks/set_repos.yml index c9fcb0c07..11863bee8 100644 --- a/ansible/roles/dnf_repos/tasks/set_repos.yml +++ b/ansible/roles/dnf_repos/tasks/set_repos.yml @@ -2,14 +2,23 @@ - name: Replace system repos with Pulp repos ansible.builtin.yum_repository: - file: "{{ item.file }}" - name: "{{ item.name }}" - baseurl: "{{ item.base_url }}" - description: "{{ item.name }}" + file: "{{ repo_values.repo_file }}" + name: "{{ repo_name }}" + baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}" + description: "{{ repo_name }}" username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" gpgcheck: false - loop: "{{ dnf_repos_repolist }}" + loop: "{{ dnf_repos_default | combine(dnf_repos_extra) | dict2items }}" + loop_control: + label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" + vars: + repo_name: "{{ item.key }}" + repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}" + repo_values: "{{ item.value[repo_os] }}" + repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}" + +- meta: end_here - name: Install epel-release # done so that roles installing epel via epel-release don't over-write our changes to the epel repo diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml index 455c26005..f7ce27ea0 100644 --- a/environments/common/inventory/group_vars/all/timestamps.yml +++ b/environments/common/inventory/group_vars/all/timestamps.yml @@ -1,88 +1,95 @@ -appliances_pulp_repos: - appstream: - '8.10': - path: rocky/8.10/AppStream/x86_64/os - timestamp: 20250614T013846 +dnf_repos_default: + appstream: # yum_repository:name + '8.10': # ansible_distribution_version or ansible_distribution_major_version + repo_file: Rocky-AppStream # yum_repository: file + pulp_path: rocky/8.10/AppStream/x86_64/os + pulp_timestamp: 20250614T013846 + # pulp_content_url: # optional '9.4': - path: rocky/9.4/AppStream/x86_64/os - timestamp: 20241112T003151 + repo_file: rocky + pulp_path: rocky/9.4/AppStream/x86_64/os + pulp_timestamp: 20241112T003151 '9.5': - path: rocky/9.5/AppStream/x86_64/os - timestamp: 20250514T014704 + repo_file: rocky + pulp_path: rocky/9.5/AppStream/x86_64/os + pulp_timestamp: 20250514T014704 '9.6': - path: rocky/9.6/AppStream/x86_64/os - timestamp: 20250726T040613 - baseos: - '8.10': - path: rocky/8.10/BaseOS/x86_64/os - timestamp: 20250614T013846 - '9.4': - path: rocky/9.4/BaseOS/x86_64/os - timestamp: 20241115T011711 - '9.5': - path: rocky/9.5/BaseOS/x86_64/os - timestamp: 20250513T031844 - '9.6': - path: rocky/9.6/BaseOS/x86_64/os - timestamp: 20250726T052250 - ceph: - '8': - path: centos/8-stream/storage/x86_64/ceph-quincy - timestamp: 20231104T015751 - '9': - path: centos/9-stream/storage/x86_64/ceph-reef - timestamp: 20250617T023108 - crb: - '8.10': - path: rocky/8.10/PowerTools/x86_64/os - timestamp: 20250614T013846 - '9.4': - path: rocky/9.4/CRB/x86_64/os - timestamp: 20241115T003133 - '9.5': - path: rocky/9.5/CRB/x86_64/os - timestamp: 20250514T014704 - '9.6': - path: rocky/9.6/CRB/x86_64/os - timestamp: 20250726T040613 - epel: - '8': - path: epel/8/Everything/x86_64 - timestamp: 20250615T234151 - '9': - path: epel/9/Everything/x86_64 - timestamp: 20250729T235750 - extras: - '8.10': - path: rocky/8.10/extras/x86_64/os - timestamp: 20250510T032327 - '9.4': - path: rocky/9.4/extras/x86_64/os - timestamp: 20241118T002802 - '9.5': - path: rocky/9.5/extras/x86_64/os - timestamp: 20250506T032818 - '9.6': - path: rocky/9.6/extras/x86_64/os - timestamp: 20250726T040613 - grafana: - '8': - path: grafana/oss/rpm - timestamp: 20250615T005738 - '9': - path: grafana/oss/rpm - timestamp: 20250730T011314 - openhpc_base: - '8': - path: OpenHPC/2/EL_8 - timestamp: 20241218T154614 - '9': - path: OpenHPC/3/EL_9 - timestamp: 20241218T154614 - openhpc_updates: + repo_file: rocky + pulp_path: rocky/9.6/AppStream/x86_64/os + pulp_timestamp: 20250726T040613 + # baseos: + # '8.10': + # path: rocky/8.10/BaseOS/x86_64/os + # timestamp: 20250614T013846 + # '9.4': + # path: rocky/9.4/BaseOS/x86_64/os + # timestamp: 20241115T011711 + # '9.5': + # path: rocky/9.5/BaseOS/x86_64/os + # timestamp: 20250513T031844 + # '9.6': + # path: rocky/9.6/BaseOS/x86_64/os + # timestamp: 20250726T052250 + Ceph: '8': - path: OpenHPC/2/updates/EL_8 - timestamp: 20250512T003315 + repo_file: ceph + pulp_path: centos/8-stream/storage/x86_64/ceph-quincy + pulp_timestamp: 20231104T015751 '9': - path: OpenHPC/3/updates/EL_9 - timestamp: 20250510T003301 + repo_file: ceph + pulp_path: centos/9-stream/storage/x86_64/ceph-reef + pulp_timestamp: 20250617T023108 + # crb: + # '8.10': + # path: rocky/8.10/PowerTools/x86_64/os + # timestamp: 20250614T013846 + # '9.4': + # path: rocky/9.4/CRB/x86_64/os + # timestamp: 20241115T003133 + # '9.5': + # path: rocky/9.5/CRB/x86_64/os + # timestamp: 20250514T014704 + # '9.6': + # path: rocky/9.6/CRB/x86_64/os + # timestamp: 20250726T040613 + # epel: + # '8': + # path: epel/8/Everything/x86_64 + # timestamp: 20250615T234151 + # '9': + # path: epel/9/Everything/x86_64 + # timestamp: 20250729T235750 + # extras: + # '8.10': + # path: rocky/8.10/extras/x86_64/os + # timestamp: 20250510T032327 + # '9.4': + # path: rocky/9.4/extras/x86_64/os + # timestamp: 20241118T002802 + # '9.5': + # path: rocky/9.5/extras/x86_64/os + # timestamp: 20250506T032818 + # '9.6': + # path: rocky/9.6/extras/x86_64/os + # timestamp: 20250726T040613 + # grafana: + # '8': + # path: grafana/oss/rpm + # timestamp: 20250615T005738 + # '9': + # path: grafana/oss/rpm + # timestamp: 20250730T011314 + # openhpc_base: + # '8': + # path: OpenHPC/2/EL_8 + # timestamp: 20241218T154614 + # '9': + # path: OpenHPC/3/EL_9 + # timestamp: 20241218T154614 + # openhpc_updates: + # '8': + # path: OpenHPC/2/updates/EL_8 + # timestamp: 20250512T003315 + # '9': + # path: OpenHPC/3/updates/EL_9 + # timestamp: 20250510T003301 From 86f93ed4a60437dfb21d7bccd0d3dfc8e21d066f Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Thu, 7 Aug 2025 15:14:00 +0000 Subject: [PATCH 02/30] add more repos and cope with CRB/PowerTools oddness --- ansible/roles/dnf_repos/tasks/set_repos.yml | 3 +- .../inventory/group_vars/all/timestamps.yml | 143 ++++++++++-------- 2 files changed, 84 insertions(+), 62 deletions(-) diff --git a/ansible/roles/dnf_repos/tasks/set_repos.yml b/ansible/roles/dnf_repos/tasks/set_repos.yml index 11863bee8..209fec313 100644 --- a/ansible/roles/dnf_repos/tasks/set_repos.yml +++ b/ansible/roles/dnf_repos/tasks/set_repos.yml @@ -13,9 +13,9 @@ loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" vars: - repo_name: "{{ item.key }}" repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}" repo_values: "{{ item.value[repo_os] }}" + repo_name: "{{ repo_values.repo_name | default(item.key) }}" repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}" - meta: end_here @@ -25,6 +25,7 @@ ansible.builtin.dnf: name: epel-release +# TODO: FIXME - name: Use Pulp EPEL repo ansible.builtin.yum_repository: name: epel diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml index f7ce27ea0..51d253f95 100644 --- a/environments/common/inventory/group_vars/all/timestamps.yml +++ b/environments/common/inventory/group_vars/all/timestamps.yml @@ -2,9 +2,10 @@ dnf_repos_default: appstream: # yum_repository:name '8.10': # ansible_distribution_version or ansible_distribution_major_version repo_file: Rocky-AppStream # yum_repository: file + # repo_name: # optional, override yum_repository:name pulp_path: rocky/8.10/AppStream/x86_64/os pulp_timestamp: 20250614T013846 - # pulp_content_url: # optional + # pulp_content_url: # optional, dnf_repos_pulp_content_url '9.4': repo_file: rocky pulp_path: rocky/9.4/AppStream/x86_64/os @@ -17,19 +18,23 @@ dnf_repos_default: repo_file: rocky pulp_path: rocky/9.6/AppStream/x86_64/os pulp_timestamp: 20250726T040613 - # baseos: - # '8.10': - # path: rocky/8.10/BaseOS/x86_64/os - # timestamp: 20250614T013846 - # '9.4': - # path: rocky/9.4/BaseOS/x86_64/os - # timestamp: 20241115T011711 - # '9.5': - # path: rocky/9.5/BaseOS/x86_64/os - # timestamp: 20250513T031844 - # '9.6': - # path: rocky/9.6/BaseOS/x86_64/os - # timestamp: 20250726T052250 + baseos: + '8.10': + repo_file: Rocky-BaseOS + pulp_path: rocky/8.10/BaseOS/x86_64/os + pulp_timestamp: 20250614T013846 + '9.4': + repo_file: rocky + pulp_path: rocky/9.4/BaseOS/x86_64/os + pulp_timestamp: 20241115T011711 + '9.5': + repo_file: rocky + pulp_path: rocky/9.5/BaseOS/x86_64/os + pulp_timestamp: 20250513T031844 + '9.6': + repo_file: rocky + pulp_path: rocky/9.6/BaseOS/x86_64/os + pulp_timestamp: 20250726T052250 Ceph: '8': repo_file: ceph @@ -39,19 +44,24 @@ dnf_repos_default: repo_file: ceph pulp_path: centos/9-stream/storage/x86_64/ceph-reef pulp_timestamp: 20250617T023108 - # crb: - # '8.10': - # path: rocky/8.10/PowerTools/x86_64/os - # timestamp: 20250614T013846 - # '9.4': - # path: rocky/9.4/CRB/x86_64/os - # timestamp: 20241115T003133 - # '9.5': - # path: rocky/9.5/CRB/x86_64/os - # timestamp: 20250514T014704 - # '9.6': - # path: rocky/9.6/CRB/x86_64/os - # timestamp: 20250726T040613 + crb: + '8.10': + repo_name: powertools + repo_file: Rocky-PowerTools + pulp_path: rocky/8.10/PowerTools/x86_64/os + pulp_timestamp: 20250614T013846 + '9.4': + repo_file: rocky + pulp_path: rocky/9.4/CRB/x86_64/os + pulp_timestamp: 20241115T003133 + '9.5': + repo_file: rocky + pulp_path: rocky/9.5/CRB/x86_64/os + pulp_timestamp: 20250514T014704 + '9.6': + repo_file: rocky + pulp_path: rocky/9.6/CRB/x86_64/os + pulp_timestamp: 20250726T040613 # epel: # '8': # path: epel/8/Everything/x86_64 @@ -59,37 +69,48 @@ dnf_repos_default: # '9': # path: epel/9/Everything/x86_64 # timestamp: 20250729T235750 - # extras: - # '8.10': - # path: rocky/8.10/extras/x86_64/os - # timestamp: 20250510T032327 - # '9.4': - # path: rocky/9.4/extras/x86_64/os - # timestamp: 20241118T002802 - # '9.5': - # path: rocky/9.5/extras/x86_64/os - # timestamp: 20250506T032818 - # '9.6': - # path: rocky/9.6/extras/x86_64/os - # timestamp: 20250726T040613 - # grafana: - # '8': - # path: grafana/oss/rpm - # timestamp: 20250615T005738 - # '9': - # path: grafana/oss/rpm - # timestamp: 20250730T011314 - # openhpc_base: - # '8': - # path: OpenHPC/2/EL_8 - # timestamp: 20241218T154614 - # '9': - # path: OpenHPC/3/EL_9 - # timestamp: 20241218T154614 - # openhpc_updates: - # '8': - # path: OpenHPC/2/updates/EL_8 - # timestamp: 20250512T003315 - # '9': - # path: OpenHPC/3/updates/EL_9 - # timestamp: 20250510T003301 + extras: + '8.10': + repo_file: Rocky-Extras + pulp_path: rocky/8.10/extras/x86_64/os + pulp_timestamp: 20250510T032327 + '9.4': + repo_file: rocky-extras + pulp_path: rocky/9.4/extras/x86_64/os + pulp_timestamp: 20241118T002802 + '9.5': + repo_file: rocky-extras + pulp_path: rocky/9.5/extras/x86_64/os + pulp_timestamp: 20250506T032818 + '9.6': + repo_file: rocky-extras + pulp_path: rocky/9.6/extras/x86_64/os + pulp_timestamp: 20250726T040613 + grafana: + '8': + repo_file: grafana + pulp_path: grafana/oss/rpm + timestamp: 20250615T005738 + '9': + repo_file: grafana + pulp_path: grafana/oss/rpm + pulp_timestamp: 20250730T011314 + # TODO: figure out how to break out openhpc repos out separately to allow for no-ohpc case + openhpc_base: + '8': + repo_file: OpenHPC + pulp_path: OpenHPC/2/EL_8 + pulp_timestamp: 20241218T154614 + '9': + repo_file: OpenHPC + pulp_path: OpenHPC/3/EL_9 + pulp_timestamp: 20241218T154614 + openhpc_updates: + '8': + repo_file: OpenHPC + pulp_path: OpenHPC/2/updates/EL_8 + pulp_timestamp: 20250512T003315 + '9': + repo_file: OpenHPC + pulp_path: OpenHPC/3/updates/EL_9 + pulp_timestamp: 20250510T003301 From a9e6959e0b8d1c0ad0104a7b97e4f70b5aba261f Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Thu, 7 Aug 2025 15:33:56 +0000 Subject: [PATCH 03/30] add epel --- ansible/roles/dnf_repos/defaults/main.yml | 51 ------------------- ansible/roles/dnf_repos/tasks/set_repos.yml | 27 ++++++---- .../inventory/group_vars/all/timestamps.yml | 19 ++++--- 3 files changed, 29 insertions(+), 68 deletions(-) diff --git a/ansible/roles/dnf_repos/defaults/main.yml b/ansible/roles/dnf_repos/defaults/main.yml index cd19405fc..df7a9dde2 100644 --- a/ansible/roles/dnf_repos/defaults/main.yml +++ b/ansible/roles/dnf_repos/defaults/main.yml @@ -3,54 +3,3 @@ dnf_repos_extra: {} dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content" dnf_repos_username: "{{ omit }}" dnf_repos_password: "{{ omit }}" - -dnf_repos_filenames: - '8': - baseos: 'Rocky-BaseOS' - appstream: 'Rocky-AppStream' - crb: 'Rocky-PowerTools' - extras: 'Rocky-Extras' - grafana: 'grafana' - '9': - baseos: 'rocky' - appstream: 'rocky' - crb: 'rocky' - extras: 'rocky-extras' - grafana: 'grafana' - -dnf_repos_version_filenames: "{{ dnf_repos_filenames[ansible_distribution_major_version] }}" - -# epel installed separately -dnf_repos_default_repolist: -- file: "{{ dnf_repos_version_filenames.baseos }}" - name: baseos - base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.baseos[ansible_distribution_version] | appliances_repo_to_subpath }}" -- file: "{{ dnf_repos_version_filenames.appstream }}" - name: appstream - base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.appstream[ansible_distribution_version] | appliances_repo_to_subpath }}" -- file: "{{ dnf_repos_version_filenames.crb }}" - name: "{{ 'powertools' if ansible_distribution_major_version == '8' else 'crb' }}" - base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.crb[ansible_distribution_version] | appliances_repo_to_subpath }}" -- file: "{{ dnf_repos_version_filenames.extras }}" - name: extras - base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.extras[ansible_distribution_version] | appliances_repo_to_subpath }}" -- file: ceph - name: Ceph - base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.ceph[ansible_distribution_major_version] | appliances_repo_to_subpath }}" -- file: "{{ dnf_repos_version_filenames.grafana }}" - name: grafana - base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.grafana[ansible_distribution_major_version] | appliances_repo_to_subpath }}" - -dnf_repos_openhpc_repolist: -- name: OpenHPC - file: OpenHPC - base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_base[ansible_distribution_major_version] | appliances_repo_to_subpath }}" -- name: OpenHPC-updates - file: OpenHPC - base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_updates[ansible_distribution_major_version] | appliances_repo_to_subpath }}" - -dnf_repos_extra_repolist: [] -dnf_repos_repolist: "{{ dnf_repos_default_repolist + (dnf_repos_openhpc_repolist if (openhpc_install_type | default('ohpc')) == 'ohpc' else []) + dnf_repos_extra_repolist }}" - -dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.epel[ansible_distribution_major_version] | appliances_repo_to_subpath }}" -dnf_repos_epel_description: "epel" diff --git a/ansible/roles/dnf_repos/tasks/set_repos.yml b/ansible/roles/dnf_repos/tasks/set_repos.yml index 209fec313..00310c53c 100644 --- a/ansible/roles/dnf_repos/tasks/set_repos.yml +++ b/ansible/roles/dnf_repos/tasks/set_repos.yml @@ -18,20 +18,27 @@ repo_name: "{{ repo_values.repo_name | default(item.key) }}" repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}" -- meta: end_here - - name: Install epel-release - # done so that roles installing epel via epel-release don't over-write our changes to the epel repo + # So roles installing epel via epel-release don't overwrite changes to the epel repo below ansible.builtin.dnf: name: epel-release -# TODO: FIXME -- name: Use Pulp EPEL repo +- name: Replace epel repo with Pulp repo ansible.builtin.yum_repository: - name: epel - file: epel - description: "{{ dnf_repos_epel_description }}" - gpgcheck: false - baseurl: "{{ dnf_repos_epel_baseurl }}" + file: "{{ repo_values.repo_file }}" + name: "{{ repo_name }}" + baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}" + description: "{{ repo_name }}" username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" + gpgcheck: false # TODO: is this really false here and above?? + loop: "{{ dnf_repos_default_epel | dict2items }}" + loop_control: + label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" + vars: + repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}" + repo_values: "{{ item.value[repo_os] }}" + repo_name: "{{ repo_values.repo_name | default(item.key) }}" + repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}" + +- meta: end_here \ No newline at end of file diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml index 51d253f95..0fd4c4c03 100644 --- a/environments/common/inventory/group_vars/all/timestamps.yml +++ b/environments/common/inventory/group_vars/all/timestamps.yml @@ -62,13 +62,6 @@ dnf_repos_default: repo_file: rocky pulp_path: rocky/9.6/CRB/x86_64/os pulp_timestamp: 20250726T040613 - # epel: - # '8': - # path: epel/8/Everything/x86_64 - # timestamp: 20250615T234151 - # '9': - # path: epel/9/Everything/x86_64 - # timestamp: 20250729T235750 extras: '8.10': repo_file: Rocky-Extras @@ -114,3 +107,15 @@ dnf_repos_default: repo_file: OpenHPC pulp_path: OpenHPC/3/updates/EL_9 pulp_timestamp: 20250510T003301 + +dnf_repos_default_epel: # as for dnf_repos_default + epel: + '8': + repo_file: epel + pulp_path: epel/8/Everything/x86_64 + pulp_timestamp: 20250615T234151 + '9': + repo_file: epel + pulp_path: epel/9/Everything/x86_64 + pulp_timestamp: 20250729T235750 + From 15579c3e8162655bf0d83d494c3a18898772a0c6 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Thu, 7 Aug 2025 15:51:43 +0000 Subject: [PATCH 04/30] use pulp_server as a group --- ansible/adhoc/deploy-pulp.yml | 15 ++++----------- ansible/roles/pulp_site/defaults/main.yml | 2 +- docs/experimental/pulp.md | 19 +++++++++++++++---- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/ansible/adhoc/deploy-pulp.yml b/ansible/adhoc/deploy-pulp.yml index 2858d032b..98d002d7a 100644 --- a/ansible/adhoc/deploy-pulp.yml +++ b/ansible/adhoc/deploy-pulp.yml @@ -1,15 +1,8 @@ # Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=" -- name: Add temporary pulp server host - hosts: localhost - tasks: - - ansible.builtin.add_host: - name: "{{ pulp_server }}" - group: "_pulp_host" - -- name: Install pulp on server and add to config +- name: Install pulp on server become: yes - hosts: _pulp_host + hosts: pulp_server # TODO: add this to common/groups tasks: - name: Install pulp ansible.builtin.include_role: @@ -22,5 +15,5 @@ debug: msg: | Server configured, override 'appliances_pulp_url' with - appliances_pulp_url: "http://{{ pulp_server }}:{{ pulp_site_port }}" - in your environments + appliances_pulp_url: "http://{{ hostvars[groups['pulp_server'] | first].ansible_host }}:{{ pulp_site_port }}" + (or the correct IP if multi-homed) in your environments diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml index d30d1bdff..a25e06c2b 100644 --- a/ansible/roles/pulp_site/defaults/main.yml +++ b/ansible/roles/pulp_site/defaults/main.yml @@ -28,7 +28,7 @@ pulp_site_rpm_info: subpath: "{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}" - name: "ceph-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major].timestamp }}" subpath: "{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}" -- name: "grafana-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.grafana.timestamp[pulp_site_target_distribution_version_major].timestamp }} +- name: "grafana-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.grafana[pulp_site_target_distribution_version_major].timestamp }}" subpath: "{{ appliances_pulp_repos.grafana[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}" pulp_site_rpm_repo_defaults: diff --git a/docs/experimental/pulp.md b/docs/experimental/pulp.md index c6b437d20..954cec5c8 100644 --- a/docs/experimental/pulp.md +++ b/docs/experimental/pulp.md @@ -5,13 +5,24 @@ In order to ensure reproducible builds, the appliance can build images using rep ## Deploying/configuring Pulp Server ### Deploying a Pulp server -A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. This can be run with -`ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server="` -where `target_host` is any resolvable host. This will print a Pulp URL which can be copied to your environments as appropriate. Ensure that the server is accessible on the specified port. Note access to this server's content isn't authenticated so assumes the server is deployed behind a secure network. +A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. To use this, create an inventory file defining a group `pulp_server` containing a single host. The hostvar `ansible_host` should be defined, giving the IP address Ansible should use for ssh. + +**TODO: example inventory file** + +Once complete, it will print a message giving a value to set for `appliances_pulp_url`, assuming the `ansible_host` address is also the address the cluster +should use to reach the Pulp server. + +**TODO: example config** + +Note access to this server's content isn't authenticated so this assumes the `pulp_server` host is not externall reachable. + +**TODO: You can actually do this using additional_nodes now, how would we make the pulp store persistant?** ### Using an existing Pulp server An existing Pulp server can be used to host Ark repos by overriding `pulp_site_password` and `appliances_pulp_url` in the target environment. Note that this assumes the same configuration as the appliance deployed Pulp i.e no content authentication. ## Syncing Pulp content with Ark -If the `pulp` group is added to the Packer build groups, the local Pulp server will be synced with Ark on build. You must authenticate with Ark by overriding `pulp_site_upstream_username` and `pulp_site_upstream_password` with your vault encrypted Ark dev credentials. `dnf_repos_username` and `dnf_repos_password` must remain unset to access content from the local Pulp. Content can also be synced by running `ansible/adhoc/sync-pulp.yml`. By default this syncs repositories for Rocky 9.5 with x86_64 architecture, but can be overridden by setting extra variables for `pulp_site_target_arch`, `pulp_site_target_distribution`, `pulp_site_target_distribution_version` and `pulp_site_target_distribution_version_major`. +If the `pulp` group is added to the Packer build groups, the local Pulp server will be synced with Ark on build. You must authenticate with Ark by overriding `pulp_site_upstream_username` and `pulp_site_upstream_password` with your vault encrypted Ark dev credentials. `dnf_repos_username` and `dnf_repos_password` must remain unset to access content from the local Pulp. + +Content can also be synced by running `ansible/adhoc/sync-pulp.yml`. By default this syncs repositories for Rocky 9.5 but this can be overridden by setting extra variables for `pulp_site_target_arch`, `pulp_site_target_distribution`, `pulp_site_target_distribution_version` and `pulp_site_target_distribution_version_major`. From 5c39a063b933df0ee20b5e1f712674edb262b2d4 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Fri, 8 Aug 2025 14:27:58 +0000 Subject: [PATCH 05/30] add epel default --- ansible/roles/dnf_repos/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/dnf_repos/defaults/main.yml b/ansible/roles/dnf_repos/defaults/main.yml index df7a9dde2..0f67d7970 100644 --- a/ansible/roles/dnf_repos/defaults/main.yml +++ b/ansible/roles/dnf_repos/defaults/main.yml @@ -1,5 +1,6 @@ dnf_repos_default: {} # see environments/common/inventory/group_vars/all/timestamps.yml dnf_repos_extra: {} +dnf_repos_default_epel: {} dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content" dnf_repos_username: "{{ omit }}" dnf_repos_password: "{{ omit }}" From 57784b77de49c33f22dd5df01c651c628c39a06f Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Fri, 8 Aug 2025 15:31:05 +0000 Subject: [PATCH 06/30] wip: get pulp sync working --- ansible/adhoc/sync-pulp.yml | 3 +- ansible/roles/pulp_site/defaults/main.yml | 36 +++++-------- .../filter_plugins/pulp-list-filters.py | 52 +++++++++++++------ docs/experimental/pulp.md | 4 ++ .../inventory/group_vars/all/timestamps.yml | 27 ++++++++++ 5 files changed, 81 insertions(+), 41 deletions(-) diff --git a/ansible/adhoc/sync-pulp.yml b/ansible/adhoc/sync-pulp.yml index b2cd9a8c4..0dbf7b5b2 100644 --- a/ansible/adhoc/sync-pulp.yml +++ b/ansible/adhoc/sync-pulp.yml @@ -6,5 +6,4 @@ vars: pulp_site_target_arch: "x86_64" pulp_site_target_distribution: "rocky" - pulp_site_target_distribution_version: "9.5" - pulp_site_target_distribution_version_major: "9" + pulp_site_target_distribution_version: "9.5" # TODO: how can we automatically make this the latest?? diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml index a25e06c2b..2a7882ae8 100644 --- a/ansible/roles/pulp_site/defaults/main.yml +++ b/ansible/roles/pulp_site/defaults/main.yml @@ -8,28 +8,7 @@ pulp_site_validate_certs: false pulp_site_install_dir: '/home/rocky/pulp' pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}" pulp_site_target_facts: "{{ hostvars[groups['pulp'][0]]['ansible_facts'] }}" -pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}" -pulp_site_target_distribution_version_major: "{{ pulp_site_target_facts['distribution_major_version'] }}" - -pulp_site_rpm_info: -- name: "baseos-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.baseos[pulp_site_target_distribution_version].timestamp }}" - subpath: "{{ appliances_pulp_repos.baseos[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}" -- name: "appstream-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.appstream[pulp_site_target_distribution_version].timestamp }}" - subpath: "{{ appliances_pulp_repos.appstream[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}" -- name: "crb-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.crb[pulp_site_target_distribution_version].timestamp }}" - subpath: "{{ appliances_pulp_repos.crb[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}" -- name: "extras-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.extras[pulp_site_target_distribution_version].timestamp }}" - subpath: "{{ appliances_pulp_repos.extras[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}" -- name: "epel-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.epel[pulp_site_target_distribution_version_major].timestamp }}" - subpath: "{{ appliances_pulp_repos.epel[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}" -- name: "ohpc-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.openhpc_base[pulp_site_target_distribution_version_major].timestamp }}" - subpath: "{{ appliances_pulp_repos.openhpc_base[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}" -- name: "ohpc-updates-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major].timestamp }}" - subpath: "{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}" -- name: "ceph-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major].timestamp }}" - subpath: "{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}" -- name: "grafana-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.grafana[pulp_site_target_distribution_version_major].timestamp }}" - subpath: "{{ appliances_pulp_repos.grafana[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}" +pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}" # TODO: how to set automatically? pulp_site_rpm_repo_defaults: remote_username: "{{ pulp_site_upstream_username }}" @@ -37,8 +16,17 @@ pulp_site_rpm_repo_defaults: policy: on_demand state: present -_pulp_site_rpm_info_all: "{{ pulp_site_rpm_info | map('combine', pulp_site_rpm_repo_defaults) }}" +pulp_site_rpm_info: | + {{ + dnf_repos_default | + combine(dnf_repos_default_epel, dnf_repos_extra) | + select_repos(pulp_site_target_distribution_version) + }} +pulp_site_rpm_repos: | + {{ + pulp_site_rpm_info | + to_rpm_repos(pulp_site_upstream_content_url, pulp_site_rpm_repo_defaults) + }} -pulp_site_rpm_repos: "{{ _pulp_site_rpm_info_all | to_rpm_repos(pulp_site_upstream_content_url) }}" pulp_site_rpm_publications: "{{ _pulp_site_rpm_info_all | to_rpm_pubs }}" pulp_site_rpm_distributions: "{{ _pulp_site_rpm_info_all | to_rpm_distros }}" diff --git a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py index 50e912685..43295fd36 100644 --- a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py +++ b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py @@ -3,29 +3,51 @@ def filters(self): return { 'to_rpm_repos': self.to_rpm_repos, 'to_rpm_pubs': self.to_rpm_pubs, - 'to_rpm_distros': self.to_rpm_distros + 'to_rpm_distros': self.to_rpm_distros, + 'select_repos': self.select_repos, } - - def to_rpm_repos(self, list, pulp_url): - repo_list = map(lambda x: { - 'name': x['name'], - 'url': pulp_url+'/'+x['subpath'], - 'remote_username': x['remote_username'], - 'remote_password': x['remote_password'], - 'policy': x['policy'], - 'state': x['state'] }, list) - return repo_list + def select_repos(self, dnf_repos, target_distro_ver): + """ Filter dnf_repos to only those for a relevant distribution version (M.m or M). Returns a list of dicts. + TODO: note this adds distro_ver as a key + """ + + target_distro_ver_major = target_distro_ver.split('.')[0] + + rpm_repos = [] + for repokey in dnf_repos: + # select either the matching major.minor or major version: + if target_distro_ver in dnf_repos[repokey]: + selected_ver = target_distro_ver + elif target_distro_ver_major in dnf_repos[repokey]: + selected_ver = target_distro_ver_major + else: + raise ValueError(f'No key matching {target_distro_ver_major} or {target_distro_ver} found in f{repokey}') + repo_data = dnf_repos[repokey][selected_ver] + repo_data['distro_ver'] = selected_ver + rpm_repos.append(repo_data) + return rpm_repos + + def to_rpm_repos(self, rpm_info, content_url, repo_defaults): + """ TODO """ + rpm_repos = [] + for repo_data in rpm_info: + rpm_data = repo_defaults.copy() # NB: this changes behaviour vs before, so now defaults can correctly be overriden + rpm_data['name'] = f"{repo_data['pulp_repo_name']}-{repo_data['distro_ver']}-{repo_data['pulp_timestamp']}" + rpm_data['url'] = '/'.join([content_url, repo_data['pulp_path'], repo_data['pulp_timestamp']]) + rpm_repos.append(rpm_data) + return rpm_repos + def to_rpm_pubs(self, list): pub_list = map(lambda x: { - 'repository': x['name'], + 'repository': x['pulp_repo_name'], 'state': x['state'] }, list) return pub_list def to_rpm_distros(self, list): distro_list = map(lambda x: { 'name': x['name'], - 'repository': x['name'], - 'base_path': x['subpath'], + 'repository': x['pulp_repo_name'], + 'base_path': x['pulp_path'], 'state': x['state'] }, list) - return distro_list \ No newline at end of file + return distro_list diff --git a/docs/experimental/pulp.md b/docs/experimental/pulp.md index 954cec5c8..50315d8ba 100644 --- a/docs/experimental/pulp.md +++ b/docs/experimental/pulp.md @@ -7,6 +7,8 @@ In order to ensure reproducible builds, the appliance can build images using rep ### Deploying a Pulp server A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. To use this, create an inventory file defining a group `pulp_server` containing a single host. The hostvar `ansible_host` should be defined, giving the IP address Ansible should use for ssh. +**TODO: should be RL9 (or RL8?)** +**TODO: add size required** **TODO: example inventory file** Once complete, it will print a message giving a value to set for `appliances_pulp_url`, assuming the `ansible_host` address is also the address the cluster @@ -17,6 +19,8 @@ should use to reach the Pulp server. Note access to this server's content isn't authenticated so this assumes the `pulp_server` host is not externall reachable. **TODO: You can actually do this using additional_nodes now, how would we make the pulp store persistant?** +**TODO: don't advise that, we want single server for all environments** +**TODO: Add a systemd unit to run pulp!** ### Using an existing Pulp server An existing Pulp server can be used to host Ark repos by overriding `pulp_site_password` and `appliances_pulp_url` in the target environment. Note that this assumes the same configuration as the appliance deployed Pulp i.e no content authentication. diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml index 0fd4c4c03..0ee93911e 100644 --- a/environments/common/inventory/group_vars/all/timestamps.yml +++ b/environments/common/inventory/group_vars/all/timestamps.yml @@ -6,107 +6,131 @@ dnf_repos_default: pulp_path: rocky/8.10/AppStream/x86_64/os pulp_timestamp: 20250614T013846 # pulp_content_url: # optional, dnf_repos_pulp_content_url + pulp_repo_name: appstream # pulp repository name '9.4': repo_file: rocky pulp_path: rocky/9.4/AppStream/x86_64/os pulp_timestamp: 20241112T003151 + pulp_repo_name: appstream '9.5': repo_file: rocky pulp_path: rocky/9.5/AppStream/x86_64/os pulp_timestamp: 20250514T014704 + pulp_repo_name: appstream '9.6': repo_file: rocky pulp_path: rocky/9.6/AppStream/x86_64/os pulp_timestamp: 20250726T040613 + pulp_repo_name: appstream baseos: '8.10': repo_file: Rocky-BaseOS pulp_path: rocky/8.10/BaseOS/x86_64/os pulp_timestamp: 20250614T013846 + pulp_repo_name: baseos '9.4': repo_file: rocky pulp_path: rocky/9.4/BaseOS/x86_64/os pulp_timestamp: 20241115T011711 + pulp_repo_name: baseos '9.5': repo_file: rocky pulp_path: rocky/9.5/BaseOS/x86_64/os pulp_timestamp: 20250513T031844 + pulp_repo_name: baseos '9.6': repo_file: rocky pulp_path: rocky/9.6/BaseOS/x86_64/os pulp_timestamp: 20250726T052250 + pulp_repo_name: baseos Ceph: '8': repo_file: ceph pulp_path: centos/8-stream/storage/x86_64/ceph-quincy pulp_timestamp: 20231104T015751 + pulp_repo_name: baseos '9': repo_file: ceph pulp_path: centos/9-stream/storage/x86_64/ceph-reef pulp_timestamp: 20250617T023108 + pulp_repo_name: baseos crb: '8.10': repo_name: powertools repo_file: Rocky-PowerTools pulp_path: rocky/8.10/PowerTools/x86_64/os pulp_timestamp: 20250614T013846 + pulp_repo_name: crb '9.4': repo_file: rocky pulp_path: rocky/9.4/CRB/x86_64/os pulp_timestamp: 20241115T003133 + pulp_repo_name: crb '9.5': repo_file: rocky pulp_path: rocky/9.5/CRB/x86_64/os pulp_timestamp: 20250514T014704 + pulp_repo_name: crb '9.6': repo_file: rocky pulp_path: rocky/9.6/CRB/x86_64/os pulp_timestamp: 20250726T040613 + pulp_repo_name: crb extras: '8.10': repo_file: Rocky-Extras pulp_path: rocky/8.10/extras/x86_64/os pulp_timestamp: 20250510T032327 + pulp_repo_name: extras '9.4': repo_file: rocky-extras pulp_path: rocky/9.4/extras/x86_64/os pulp_timestamp: 20241118T002802 + pulp_repo_name: extras '9.5': repo_file: rocky-extras pulp_path: rocky/9.5/extras/x86_64/os pulp_timestamp: 20250506T032818 + pulp_repo_name: extras '9.6': repo_file: rocky-extras pulp_path: rocky/9.6/extras/x86_64/os pulp_timestamp: 20250726T040613 + pulp_repo_name: extras grafana: '8': repo_file: grafana pulp_path: grafana/oss/rpm timestamp: 20250615T005738 + pulp_repo_name: grafana '9': repo_file: grafana pulp_path: grafana/oss/rpm pulp_timestamp: 20250730T011314 + pulp_repo_name: grafana # TODO: figure out how to break out openhpc repos out separately to allow for no-ohpc case openhpc_base: '8': repo_file: OpenHPC pulp_path: OpenHPC/2/EL_8 pulp_timestamp: 20241218T154614 + pulp_repo_name: ohpc '9': repo_file: OpenHPC pulp_path: OpenHPC/3/EL_9 pulp_timestamp: 20241218T154614 + pulp_repo_name: ohpc openhpc_updates: '8': repo_file: OpenHPC pulp_path: OpenHPC/2/updates/EL_8 pulp_timestamp: 20250512T003315 + pulp_repo_name: ohpc-updates '9': repo_file: OpenHPC pulp_path: OpenHPC/3/updates/EL_9 pulp_timestamp: 20250510T003301 + pulp_repo_name: ohpc-updates dnf_repos_default_epel: # as for dnf_repos_default epel: @@ -114,8 +138,11 @@ dnf_repos_default_epel: # as for dnf_repos_default repo_file: epel pulp_path: epel/8/Everything/x86_64 pulp_timestamp: 20250615T234151 + pulp_repo_name: epel '9': repo_file: epel pulp_path: epel/9/Everything/x86_64 pulp_timestamp: 20250729T235750 + pulp_repo_name: epel +dnf_repos_extra: {} From 4aa9feedf6f41f2ccc1a3ef8d95c5ed2d0ea68c8 Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Fri, 15 Aug 2025 13:25:02 +0100 Subject: [PATCH 07/30] fixed sync --- ansible/adhoc/sync-pulp.yml | 2 +- ansible/roles/pulp_site/defaults/main.yml | 2 +- ansible/roles/pulp_site/tasks/sync.yml | 22 +++++++++---------- ansible/roles/pulp_site/templates/cli.toml.j2 | 1 - docs/experimental/pulp.md | 2 +- environments/common/inventory/groups | 3 +++ 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/ansible/adhoc/sync-pulp.yml b/ansible/adhoc/sync-pulp.yml index 0dbf7b5b2..adc4cb0f7 100644 --- a/ansible/adhoc/sync-pulp.yml +++ b/ansible/adhoc/sync-pulp.yml @@ -6,4 +6,4 @@ vars: pulp_site_target_arch: "x86_64" pulp_site_target_distribution: "rocky" - pulp_site_target_distribution_version: "9.5" # TODO: how can we automatically make this the latest?? + pulp_site_target_distribution_version: "9.6" # TODO: how can we automatically make this the latest?? diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml index 2a7882ae8..477fcf150 100644 --- a/ansible/roles/pulp_site/defaults/main.yml +++ b/ansible/roles/pulp_site/defaults/main.yml @@ -1,7 +1,7 @@ pulp_site_url: "{{ appliances_pulp_url }}" pulp_site_port: 8080 pulp_site_username: admin # shouldn't be changed -pulp_site_password: "{{ vault_pulp_admin_password }}" +pulp_site_password: "{{ vault_pulp_admin_password }}" #todo make more obvious this is different from the password needed for ark (pulp_site_upstream_password) pulp_site_upstream_content_url: https://ark.stackhpc.com/pulp/content pulp_site_default_upstream_suffix: "{{ pulp_site_target_arch }}/os" pulp_site_validate_certs: false diff --git a/ansible/roles/pulp_site/tasks/sync.yml b/ansible/roles/pulp_site/tasks/sync.yml index 5ef2bc5f1..9a2a93210 100644 --- a/ansible/roles/pulp_site/tasks/sync.yml +++ b/ansible/roles/pulp_site/tasks/sync.yml @@ -3,17 +3,7 @@ - ansible.builtin.assert: that: pulp_site_upstream_password != '' quiet: true - fail_msg: "Upstream password not set. Either set env var ARK_PASSWORD or override pulp_site_upstream_password." - -- name: Wait for Pulp server - pulp.squeezer.status: - pulp_url: "{{ pulp_site_url }}" - username: "{{ pulp_site_username }}" - password: "{{ pulp_site_password }}" - register: _pulp_status - until: _pulp_status.failed == false - retries: 30 - delay: 20 + fail_msg: "Upstream password not set. Ensure `pulp_site_upstream_username` and `pulp_site_upstream_password` are overriden to your Ark credentials." - name: Ensure Pulp CLI config directory exists ansible.builtin.file: @@ -27,6 +17,16 @@ dest: ~/.config/pulp/cli.toml mode: '0644' +- name: Wait for Pulp server + pulp.squeezer.status: + pulp_url: "{{ pulp_site_url }}" + username: "{{ pulp_site_username }}" + password: "{{ pulp_site_password }}" + register: _pulp_status + until: _pulp_status.failed == false + retries: 30 + delay: 20 + - block: - name: Ensure squeezer cache exists ansible.builtin.file: diff --git a/ansible/roles/pulp_site/templates/cli.toml.j2 b/ansible/roles/pulp_site/templates/cli.toml.j2 index 06867902f..c67dcf393 100644 --- a/ansible/roles/pulp_site/templates/cli.toml.j2 +++ b/ansible/roles/pulp_site/templates/cli.toml.j2 @@ -4,7 +4,6 @@ username = "{{ pulp_site_username }}" password = "{{ pulp_site_password }}" api_root = "/pulp/" domain = "default" -headers = [] cert = "" key = "" verify_ssl = true diff --git a/docs/experimental/pulp.md b/docs/experimental/pulp.md index 50315d8ba..ee2580403 100644 --- a/docs/experimental/pulp.md +++ b/docs/experimental/pulp.md @@ -8,7 +8,7 @@ In order to ensure reproducible builds, the appliance can build images using rep A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. To use this, create an inventory file defining a group `pulp_server` containing a single host. The hostvar `ansible_host` should be defined, giving the IP address Ansible should use for ssh. **TODO: should be RL9 (or RL8?)** -**TODO: add size required** +**TODO: add size required (2 vCPUs, 4GB RAM)** **TODO: example inventory file** Once complete, it will print a message giving a value to set for `appliances_pulp_url`, assuming the `ansible_host` address is also the address the cluster diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups index 57b644152..3b567df3a 100644 --- a/environments/common/inventory/groups +++ b/environments/common/inventory/groups @@ -211,3 +211,6 @@ extra_packages [nhc] # Hosts to configure for node health checks - either entire 'compute' group or empty + +[pulp_server] + From dd52e90789a8e9090297fdd2ce5cf34daa181b2b Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Fri, 15 Aug 2025 16:33:20 +0100 Subject: [PATCH 08/30] autodetect latest in adhoc script, refactored timestamps to allow gated ohpc repos, fixed pulp site --- ansible/adhoc/sync-pulp.yml | 2 +- ansible/roles/dnf_repos/defaults/main.yml | 2 + ansible/roles/dnf_repos/tasks/set_repos.yml | 4 +- ansible/roles/pulp_site/defaults/main.yml | 8 +- .../filter_plugins/pulp-list-filters.py | 18 +- .../inventory/group_vars/all/timestamps.yml | 295 +++++++++--------- 6 files changed, 169 insertions(+), 160 deletions(-) diff --git a/ansible/adhoc/sync-pulp.yml b/ansible/adhoc/sync-pulp.yml index adc4cb0f7..c879d438b 100644 --- a/ansible/adhoc/sync-pulp.yml +++ b/ansible/adhoc/sync-pulp.yml @@ -6,4 +6,4 @@ vars: pulp_site_target_arch: "x86_64" pulp_site_target_distribution: "rocky" - pulp_site_target_distribution_version: "9.6" # TODO: how can we automatically make this the latest?? + pulp_site_target_distribution_version: "{{ dnf_repos_all['baseos'].keys() | map('float') | sort | last }}" \ No newline at end of file diff --git a/ansible/roles/dnf_repos/defaults/main.yml b/ansible/roles/dnf_repos/defaults/main.yml index 0f67d7970..aa9757e57 100644 --- a/ansible/roles/dnf_repos/defaults/main.yml +++ b/ansible/roles/dnf_repos/defaults/main.yml @@ -1,5 +1,7 @@ dnf_repos_default: {} # see environments/common/inventory/group_vars/all/timestamps.yml dnf_repos_extra: {} +dnf_repos_no_epel: {} +dnf_repos_all: {} dnf_repos_default_epel: {} dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content" dnf_repos_username: "{{ omit }}" diff --git a/ansible/roles/dnf_repos/tasks/set_repos.yml b/ansible/roles/dnf_repos/tasks/set_repos.yml index 00310c53c..d35bfe260 100644 --- a/ansible/roles/dnf_repos/tasks/set_repos.yml +++ b/ansible/roles/dnf_repos/tasks/set_repos.yml @@ -9,7 +9,7 @@ username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" gpgcheck: false - loop: "{{ dnf_repos_default | combine(dnf_repos_extra) | dict2items }}" + loop: "{{ dnf_repos_no_epel | dict2items }}" loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" vars: @@ -40,5 +40,3 @@ repo_values: "{{ item.value[repo_os] }}" repo_name: "{{ repo_values.repo_name | default(item.key) }}" repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}" - -- meta: end_here \ No newline at end of file diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml index 477fcf150..c2c269baa 100644 --- a/ansible/roles/pulp_site/defaults/main.yml +++ b/ansible/roles/pulp_site/defaults/main.yml @@ -14,12 +14,10 @@ pulp_site_rpm_repo_defaults: remote_username: "{{ pulp_site_upstream_username }}" remote_password: "{{ pulp_site_upstream_password }}" policy: on_demand - state: present pulp_site_rpm_info: | {{ - dnf_repos_default | - combine(dnf_repos_default_epel, dnf_repos_extra) | + dnf_repos_all | select_repos(pulp_site_target_distribution_version) }} pulp_site_rpm_repos: | @@ -28,5 +26,5 @@ pulp_site_rpm_repos: | to_rpm_repos(pulp_site_upstream_content_url, pulp_site_rpm_repo_defaults) }} -pulp_site_rpm_publications: "{{ _pulp_site_rpm_info_all | to_rpm_pubs }}" -pulp_site_rpm_distributions: "{{ _pulp_site_rpm_info_all | to_rpm_distros }}" +pulp_site_rpm_publications: "{{ pulp_site_rpm_info | to_rpm_pubs }}" +pulp_site_rpm_distributions: "{{ pulp_site_rpm_info | to_rpm_distros }}" diff --git a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py index 43295fd36..9b7225f34 100644 --- a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py +++ b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py @@ -7,7 +7,7 @@ def filters(self): 'select_repos': self.select_repos, } - def select_repos(self, dnf_repos, target_distro_ver): + def select_repos(self, dnf_repos, target_distro_ver): #TODO: why does baseos get a major and minor version? """ Filter dnf_repos to only those for a relevant distribution version (M.m or M). Returns a list of dicts. TODO: note this adds distro_ver as a key """ @@ -33,21 +33,25 @@ def to_rpm_repos(self, rpm_info, content_url, repo_defaults): rpm_repos = [] for repo_data in rpm_info: rpm_data = repo_defaults.copy() # NB: this changes behaviour vs before, so now defaults can correctly be overriden - rpm_data['name'] = f"{repo_data['pulp_repo_name']}-{repo_data['distro_ver']}-{repo_data['pulp_timestamp']}" + rpm_data['name'] = get_repo_name(repo_data) rpm_data['url'] = '/'.join([content_url, repo_data['pulp_path'], repo_data['pulp_timestamp']]) + rpm_data['state'] = 'present' rpm_repos.append(rpm_data) return rpm_repos def to_rpm_pubs(self, list): pub_list = map(lambda x: { - 'repository': x['pulp_repo_name'], - 'state': x['state'] }, list) + 'repository': get_repo_name(x), + 'state': 'present' }, list) return pub_list def to_rpm_distros(self, list): distro_list = map(lambda x: { - 'name': x['name'], - 'repository': x['pulp_repo_name'], + 'name': x['pulp_repo_name'], + 'repository': get_repo_name(x), 'base_path': x['pulp_path'], - 'state': x['state'] }, list) + 'state': 'present' }, list) return distro_list + +def get_repo_name(dnf_repos_data): + return f"{dnf_repos_data['pulp_repo_name']}-{dnf_repos_data['distro_ver']}-{dnf_repos_data['pulp_timestamp']}" diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml index 0ee93911e..2effd1080 100644 --- a/environments/common/inventory/group_vars/all/timestamps.yml +++ b/environments/common/inventory/group_vars/all/timestamps.yml @@ -1,148 +1,155 @@ -dnf_repos_default: - appstream: # yum_repository:name - '8.10': # ansible_distribution_version or ansible_distribution_major_version - repo_file: Rocky-AppStream # yum_repository: file - # repo_name: # optional, override yum_repository:name - pulp_path: rocky/8.10/AppStream/x86_64/os - pulp_timestamp: 20250614T013846 - # pulp_content_url: # optional, dnf_repos_pulp_content_url - pulp_repo_name: appstream # pulp repository name - '9.4': - repo_file: rocky - pulp_path: rocky/9.4/AppStream/x86_64/os - pulp_timestamp: 20241112T003151 - pulp_repo_name: appstream - '9.5': - repo_file: rocky - pulp_path: rocky/9.5/AppStream/x86_64/os - pulp_timestamp: 20250514T014704 - pulp_repo_name: appstream - '9.6': - repo_file: rocky - pulp_path: rocky/9.6/AppStream/x86_64/os - pulp_timestamp: 20250726T040613 - pulp_repo_name: appstream - baseos: - '8.10': - repo_file: Rocky-BaseOS - pulp_path: rocky/8.10/BaseOS/x86_64/os - pulp_timestamp: 20250614T013846 - pulp_repo_name: baseos - '9.4': - repo_file: rocky - pulp_path: rocky/9.4/BaseOS/x86_64/os - pulp_timestamp: 20241115T011711 - pulp_repo_name: baseos - '9.5': - repo_file: rocky - pulp_path: rocky/9.5/BaseOS/x86_64/os - pulp_timestamp: 20250513T031844 - pulp_repo_name: baseos - '9.6': - repo_file: rocky - pulp_path: rocky/9.6/BaseOS/x86_64/os - pulp_timestamp: 20250726T052250 - pulp_repo_name: baseos - Ceph: - '8': - repo_file: ceph - pulp_path: centos/8-stream/storage/x86_64/ceph-quincy - pulp_timestamp: 20231104T015751 - pulp_repo_name: baseos - '9': - repo_file: ceph - pulp_path: centos/9-stream/storage/x86_64/ceph-reef - pulp_timestamp: 20250617T023108 - pulp_repo_name: baseos - crb: - '8.10': - repo_name: powertools - repo_file: Rocky-PowerTools - pulp_path: rocky/8.10/PowerTools/x86_64/os - pulp_timestamp: 20250614T013846 - pulp_repo_name: crb - '9.4': - repo_file: rocky - pulp_path: rocky/9.4/CRB/x86_64/os - pulp_timestamp: 20241115T003133 - pulp_repo_name: crb - '9.5': - repo_file: rocky - pulp_path: rocky/9.5/CRB/x86_64/os - pulp_timestamp: 20250514T014704 - pulp_repo_name: crb - '9.6': - repo_file: rocky - pulp_path: rocky/9.6/CRB/x86_64/os - pulp_timestamp: 20250726T040613 - pulp_repo_name: crb - extras: - '8.10': - repo_file: Rocky-Extras - pulp_path: rocky/8.10/extras/x86_64/os - pulp_timestamp: 20250510T032327 - pulp_repo_name: extras - '9.4': - repo_file: rocky-extras - pulp_path: rocky/9.4/extras/x86_64/os - pulp_timestamp: 20241118T002802 - pulp_repo_name: extras - '9.5': - repo_file: rocky-extras - pulp_path: rocky/9.5/extras/x86_64/os - pulp_timestamp: 20250506T032818 - pulp_repo_name: extras - '9.6': - repo_file: rocky-extras - pulp_path: rocky/9.6/extras/x86_64/os - pulp_timestamp: 20250726T040613 - pulp_repo_name: extras - grafana: - '8': - repo_file: grafana - pulp_path: grafana/oss/rpm - timestamp: 20250615T005738 - pulp_repo_name: grafana - '9': - repo_file: grafana - pulp_path: grafana/oss/rpm - pulp_timestamp: 20250730T011314 - pulp_repo_name: grafana - # TODO: figure out how to break out openhpc repos out separately to allow for no-ohpc case - openhpc_base: - '8': - repo_file: OpenHPC - pulp_path: OpenHPC/2/EL_8 - pulp_timestamp: 20241218T154614 - pulp_repo_name: ohpc - '9': - repo_file: OpenHPC - pulp_path: OpenHPC/3/EL_9 - pulp_timestamp: 20241218T154614 - pulp_repo_name: ohpc - openhpc_updates: - '8': - repo_file: OpenHPC - pulp_path: OpenHPC/2/updates/EL_8 - pulp_timestamp: 20250512T003315 - pulp_repo_name: ohpc-updates - '9': - repo_file: OpenHPC - pulp_path: OpenHPC/3/updates/EL_9 - pulp_timestamp: 20250510T003301 - pulp_repo_name: ohpc-updates +dnf_repos_all: "{{ dnf_repos_no_epel | combine(dnf_repos_default_epel) }}" +dnf_repos_no_epel: | + {{ dnf_repos_default['base'] + | combine(dnf_repos_default['ohpc'] if (openhpc_install_type | default('ohpc')) == 'ohpc' else {}) + | combine(dnf_repos_extra) }} +dnf_repos_default_epel: "{{ dnf_repos_default['epel'] }}" -dnf_repos_default_epel: # as for dnf_repos_default +dnf_repos_default: + base: # top level keys for internal indexing only, see ansible/roles/dnf_repos/defaults.yml + appstream: # yum_repository:name + '8.10': # ansible_distribution_version or ansible_distribution_major_version + repo_file: Rocky-AppStream # yum_repository: file + # repo_name: # optional, override yum_repository:name + pulp_path: rocky/8.10/AppStream/x86_64/os + pulp_timestamp: 20250614T013846 + # pulp_content_url: # optional, dnf_repos_pulp_content_url + pulp_repo_name: appstream # pulp repository name + '9.4': + repo_file: rocky + pulp_path: rocky/9.4/AppStream/x86_64/os + pulp_timestamp: 20241112T003151 + pulp_repo_name: appstream + '9.5': + repo_file: rocky + pulp_path: rocky/9.5/AppStream/x86_64/os + pulp_timestamp: 20250514T014704 + pulp_repo_name: appstream + '9.6': + repo_file: rocky + pulp_path: rocky/9.6/AppStream/x86_64/os + pulp_timestamp: 20250726T040613 + pulp_repo_name: appstream + baseos: + '8.10': + repo_file: Rocky-BaseOS + pulp_path: rocky/8.10/BaseOS/x86_64/os + pulp_timestamp: 20250614T013846 + pulp_repo_name: baseos + '9.4': + repo_file: rocky + pulp_path: rocky/9.4/BaseOS/x86_64/os + pulp_timestamp: 20241115T011711 + pulp_repo_name: baseos + '9.5': + repo_file: rocky + pulp_path: rocky/9.5/BaseOS/x86_64/os + pulp_timestamp: 20250513T031844 + pulp_repo_name: baseos + '9.6': + repo_file: rocky + pulp_path: rocky/9.6/BaseOS/x86_64/os + pulp_timestamp: 20250726T052250 + pulp_repo_name: baseos + Ceph: + '8': + repo_file: ceph + pulp_path: centos/8-stream/storage/x86_64/ceph-quincy + pulp_timestamp: 20231104T015751 + pulp_repo_name: ceph + '9': + repo_file: ceph + pulp_path: centos/9-stream/storage/x86_64/ceph-reef + pulp_timestamp: 20250617T023108 + pulp_repo_name: ceph + crb: + '8.10': + repo_name: powertools + repo_file: Rocky-PowerTools + pulp_path: rocky/8.10/PowerTools/x86_64/os + pulp_timestamp: 20250614T013846 + pulp_repo_name: crb + '9.4': + repo_file: rocky + pulp_path: rocky/9.4/CRB/x86_64/os + pulp_timestamp: 20241115T003133 + pulp_repo_name: crb + '9.5': + repo_file: rocky + pulp_path: rocky/9.5/CRB/x86_64/os + pulp_timestamp: 20250514T014704 + pulp_repo_name: crb + '9.6': + repo_file: rocky + pulp_path: rocky/9.6/CRB/x86_64/os + pulp_timestamp: 20250726T040613 + pulp_repo_name: crb + extras: + '8.10': + repo_file: Rocky-Extras + pulp_path: rocky/8.10/extras/x86_64/os + pulp_timestamp: 20250510T032327 + pulp_repo_name: extras + '9.4': + repo_file: rocky-extras + pulp_path: rocky/9.4/extras/x86_64/os + pulp_timestamp: 20241118T002802 + pulp_repo_name: extras + '9.5': + repo_file: rocky-extras + pulp_path: rocky/9.5/extras/x86_64/os + pulp_timestamp: 20250506T032818 + pulp_repo_name: extras + '9.6': + repo_file: rocky-extras + pulp_path: rocky/9.6/extras/x86_64/os + pulp_timestamp: 20250726T040613 + pulp_repo_name: extras + grafana: + '8': + repo_file: grafana + pulp_path: grafana/oss/rpm + timestamp: 20250615T005738 + pulp_repo_name: grafana + '9': + repo_file: grafana + pulp_path: grafana/oss/rpm + pulp_timestamp: 20250730T011314 + pulp_repo_name: grafana + ohpc: + openhpc_base: + '8': + repo_file: OpenHPC + pulp_path: OpenHPC/2/EL_8 + pulp_timestamp: 20241218T154614 + pulp_repo_name: ohpc + '9': + repo_file: OpenHPC + pulp_path: OpenHPC/3/EL_9 + pulp_timestamp: 20241218T154614 + pulp_repo_name: ohpc + openhpc_updates: + '8': + repo_file: OpenHPC + pulp_path: OpenHPC/2/updates/EL_8 + pulp_timestamp: 20250512T003315 + pulp_repo_name: ohpc-updates + '9': + repo_file: OpenHPC + pulp_path: OpenHPC/3/updates/EL_9 + pulp_timestamp: 20250510T003301 + pulp_repo_name: ohpc-updates epel: - '8': - repo_file: epel - pulp_path: epel/8/Everything/x86_64 - pulp_timestamp: 20250615T234151 - pulp_repo_name: epel - '9': - repo_file: epel - pulp_path: epel/9/Everything/x86_64 - pulp_timestamp: 20250729T235750 - pulp_repo_name: epel + epel: + '8': + repo_file: epel + pulp_path: epel/8/Everything/x86_64 + pulp_timestamp: 20250615T234151 + pulp_repo_name: epel + '9': + repo_file: epel + pulp_path: epel/9/Everything/x86_64 + pulp_timestamp: 20250729T235750 + pulp_repo_name: epel dnf_repos_extra: {} From 485490387513532930a4902caf58a653992ddfdc Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Mon, 18 Aug 2025 08:54:59 +0100 Subject: [PATCH 09/30] fixed distributions + ohpc repos --- ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py | 2 +- environments/common/inventory/group_vars/all/timestamps.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py index 9b7225f34..f903d5385 100644 --- a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py +++ b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py @@ -49,7 +49,7 @@ def to_rpm_distros(self, list): distro_list = map(lambda x: { 'name': x['pulp_repo_name'], 'repository': get_repo_name(x), - 'base_path': x['pulp_path'], + 'base_path': '/'.join([x['pulp_path'],x['pulp_timestamp']]), 'state': 'present' }, list) return distro_list diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml index 2effd1080..0f9bd7bc4 100644 --- a/environments/common/inventory/group_vars/all/timestamps.yml +++ b/environments/common/inventory/group_vars/all/timestamps.yml @@ -117,7 +117,7 @@ dnf_repos_default: pulp_timestamp: 20250730T011314 pulp_repo_name: grafana ohpc: - openhpc_base: + OpenHPC: '8': repo_file: OpenHPC pulp_path: OpenHPC/2/EL_8 @@ -128,7 +128,7 @@ dnf_repos_default: pulp_path: OpenHPC/3/EL_9 pulp_timestamp: 20241218T154614 pulp_repo_name: ohpc - openhpc_updates: + OpenHPC-updates: '8': repo_file: OpenHPC pulp_path: OpenHPC/2/updates/EL_8 From c8635bba5b27061784221a8f4d9fd452a260d12b Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Mon, 18 Aug 2025 09:43:47 +0100 Subject: [PATCH 10/30] updated timestamps script + bumped rocky 9 timestamps --- ansible/ci/update_timestamps.yml | 4 +- ansible/library/latest_timestamps.py | 23 +-- .../inventory/group_vars/all/dnf_repos.yml | 17 ++ .../inventory/group_vars/all/timestamps.yml | 154 ++++++++---------- 4 files changed, 104 insertions(+), 94 deletions(-) create mode 100644 environments/common/inventory/group_vars/all/dnf_repos.yml diff --git a/ansible/ci/update_timestamps.yml b/ansible/ci/update_timestamps.yml index e9a455a1e..b9ab9e0ec 100644 --- a/ansible/ci/update_timestamps.yml +++ b/ansible/ci/update_timestamps.yml @@ -2,7 +2,7 @@ tasks: - name: Get latest timestamps from sources latest_timestamps: - repos_dict: "{{ appliances_pulp_repos }}" + repos_dict: "{{ dnf_repos_default }}" content_url: "https://ark.stackhpc.com/pulp/content" register: _result @@ -13,4 +13,4 @@ backup: true vars: repo_template: - appliances_pulp_repos: "{{ _result.timestamps }}" + dnf_repos_default: "{{ _result.timestamps }}" diff --git a/ansible/library/latest_timestamps.py b/ansible/library/latest_timestamps.py index 6407ef049..fc82ab8cf 100644 --- a/ansible/library/latest_timestamps.py +++ b/ansible/library/latest_timestamps.py @@ -52,18 +52,21 @@ def run_module(): ) timestamps = dict(module.params['repos_dict']) - for repo in timestamps: - for version in timestamps[repo]: + for index in timestamps: + for repo in timestamps[index]: + for version in timestamps[index][repo]: - html_txt = requests.get( - url= module.params['content_url'] + '/' + timestamps[repo][version]['path'] - ).text - timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html - timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags - latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed - timestamps[repo][version]['timestamp'] = latest_timestamp + html_txt = requests.get( + url= module.params['content_url'] + '/' + timestamps[index][repo][version]['pulp_path'] + ).text + timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html + timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags + latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed + timestamps[index][repo][version]['pulp_timestamp'] = latest_timestamp + # timestamps[index] = dict(sorted(timestamps[index])) - result['timestamps'] = dict(sorted(timestamps.items())) + + result['timestamps'] = timestamps module.exit_json(**result) diff --git a/environments/common/inventory/group_vars/all/dnf_repos.yml b/environments/common/inventory/group_vars/all/dnf_repos.yml new file mode 100644 index 000000000..05514e5b1 --- /dev/null +++ b/environments/common/inventory/group_vars/all/dnf_repos.yml @@ -0,0 +1,17 @@ +dnf_repos_all: "{{ dnf_repos_no_epel | combine(dnf_repos_default_epel) }}" #see timestamps.yml +dnf_repos_no_epel: | + {{ dnf_repos_default['base'] + | combine(dnf_repos_default['ohpc'] if (openhpc_install_type | default('ohpc')) == 'ohpc' else {}) + | combine(dnf_repos_extra) }} +dnf_repos_default_epel: "{{ dnf_repos_default['epel'] }}" +dnf_repos_extra: {} +# dnf_repos_default: +# base: # top level keys for internal indexing only, see `dnf_repos_all` and `dnf_repos_no_epel` +# appstream: # yum_repository:name +# '8.10': # ansible_distribution_version or ansible_distribution_major_version +# repo_file: Rocky-AppStream # yum_repository: file +# # repo_name: # optional, override yum_repository:name +# pulp_path: rocky/8.10/AppStream/x86_64/os +# pulp_timestamp: 20250614T013846 +# # pulp_content_url: # optional, dnf_repos_pulp_content_url +# pulp_repo_name: appstream # pulp repository name \ No newline at end of file diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml index 0f9bd7bc4..4589a9c2d 100644 --- a/environments/common/inventory/group_vars/all/timestamps.yml +++ b/environments/common/inventory/group_vars/all/timestamps.yml @@ -1,155 +1,145 @@ -dnf_repos_all: "{{ dnf_repos_no_epel | combine(dnf_repos_default_epel) }}" -dnf_repos_no_epel: | - {{ dnf_repos_default['base'] - | combine(dnf_repos_default['ohpc'] if (openhpc_install_type | default('ohpc')) == 'ohpc' else {}) - | combine(dnf_repos_extra) }} -dnf_repos_default_epel: "{{ dnf_repos_default['epel'] }}" - dnf_repos_default: - base: # top level keys for internal indexing only, see ansible/roles/dnf_repos/defaults.yml - appstream: # yum_repository:name - '8.10': # ansible_distribution_version or ansible_distribution_major_version - repo_file: Rocky-AppStream # yum_repository: file - # repo_name: # optional, override yum_repository:name + base: + Ceph: + '8': + pulp_path: centos/8-stream/storage/x86_64/ceph-quincy + pulp_repo_name: ceph + pulp_timestamp: 20231104T015751 + repo_file: ceph + '9': + pulp_path: centos/9-stream/storage/x86_64/ceph-reef + pulp_repo_name: ceph + pulp_timestamp: 20250617T023108 + repo_file: ceph + appstream: + '8.10': pulp_path: rocky/8.10/AppStream/x86_64/os + pulp_repo_name: appstream pulp_timestamp: 20250614T013846 - # pulp_content_url: # optional, dnf_repos_pulp_content_url - pulp_repo_name: appstream # pulp repository name + repo_file: Rocky-AppStream '9.4': - repo_file: rocky pulp_path: rocky/9.4/AppStream/x86_64/os - pulp_timestamp: 20241112T003151 pulp_repo_name: appstream - '9.5': + pulp_timestamp: 20241112T003151 repo_file: rocky + '9.5': pulp_path: rocky/9.5/AppStream/x86_64/os - pulp_timestamp: 20250514T014704 pulp_repo_name: appstream - '9.6': + pulp_timestamp: 20250514T014704 repo_file: rocky + '9.6': pulp_path: rocky/9.6/AppStream/x86_64/os - pulp_timestamp: 20250726T040613 pulp_repo_name: appstream + pulp_timestamp: 20250816T020215 + repo_file: rocky baseos: '8.10': - repo_file: Rocky-BaseOS pulp_path: rocky/8.10/BaseOS/x86_64/os - pulp_timestamp: 20250614T013846 pulp_repo_name: baseos + pulp_timestamp: 20250614T013846 + repo_file: Rocky-BaseOS '9.4': - repo_file: rocky pulp_path: rocky/9.4/BaseOS/x86_64/os - pulp_timestamp: 20241115T011711 pulp_repo_name: baseos - '9.5': + pulp_timestamp: 20241115T011711 repo_file: rocky + '9.5': pulp_path: rocky/9.5/BaseOS/x86_64/os - pulp_timestamp: 20250513T031844 pulp_repo_name: baseos - '9.6': + pulp_timestamp: 20250513T031844 repo_file: rocky + '9.6': pulp_path: rocky/9.6/BaseOS/x86_64/os - pulp_timestamp: 20250726T052250 pulp_repo_name: baseos - Ceph: - '8': - repo_file: ceph - pulp_path: centos/8-stream/storage/x86_64/ceph-quincy - pulp_timestamp: 20231104T015751 - pulp_repo_name: ceph - '9': - repo_file: ceph - pulp_path: centos/9-stream/storage/x86_64/ceph-reef - pulp_timestamp: 20250617T023108 - pulp_repo_name: ceph + pulp_timestamp: 20250815T050653 + repo_file: rocky crb: '8.10': - repo_name: powertools - repo_file: Rocky-PowerTools pulp_path: rocky/8.10/PowerTools/x86_64/os - pulp_timestamp: 20250614T013846 pulp_repo_name: crb + pulp_timestamp: 20250614T013846 + repo_file: Rocky-PowerTools + repo_name: powertools '9.4': - repo_file: rocky pulp_path: rocky/9.4/CRB/x86_64/os - pulp_timestamp: 20241115T003133 pulp_repo_name: crb - '9.5': + pulp_timestamp: 20241115T003133 repo_file: rocky + '9.5': pulp_path: rocky/9.5/CRB/x86_64/os - pulp_timestamp: 20250514T014704 pulp_repo_name: crb - '9.6': + pulp_timestamp: 20250514T014704 repo_file: rocky + '9.6': pulp_path: rocky/9.6/CRB/x86_64/os - pulp_timestamp: 20250726T040613 pulp_repo_name: crb + pulp_timestamp: 20250815T034418 + repo_file: rocky extras: '8.10': - repo_file: Rocky-Extras pulp_path: rocky/8.10/extras/x86_64/os - pulp_timestamp: 20250510T032327 pulp_repo_name: extras + pulp_timestamp: 20250510T032327 + repo_file: Rocky-Extras '9.4': - repo_file: rocky-extras pulp_path: rocky/9.4/extras/x86_64/os - pulp_timestamp: 20241118T002802 pulp_repo_name: extras - '9.5': + pulp_timestamp: 20241118T002802 repo_file: rocky-extras + '9.5': pulp_path: rocky/9.5/extras/x86_64/os - pulp_timestamp: 20250506T032818 pulp_repo_name: extras - '9.6': + pulp_timestamp: 20250506T032818 repo_file: rocky-extras + '9.6': pulp_path: rocky/9.6/extras/x86_64/os - pulp_timestamp: 20250726T040613 pulp_repo_name: extras + pulp_timestamp: 20250726T040613 + repo_file: rocky-extras grafana: '8': - repo_file: grafana pulp_path: grafana/oss/rpm - timestamp: 20250615T005738 pulp_repo_name: grafana - '9': + pulp_timestamp: 20250730T011314 repo_file: grafana + timestamp: 20250615T005738 + '9': pulp_path: grafana/oss/rpm - pulp_timestamp: 20250730T011314 pulp_repo_name: grafana + pulp_timestamp: 20250730T011314 + repo_file: grafana + epel: + epel: + '8': + pulp_path: epel/8/Everything/x86_64 + pulp_repo_name: epel + pulp_timestamp: 20250615T234151 + repo_file: epel + '9': + pulp_path: epel/9/Everything/x86_64 + pulp_repo_name: epel + pulp_timestamp: 20250817T000753 + repo_file: epel ohpc: OpenHPC: '8': - repo_file: OpenHPC pulp_path: OpenHPC/2/EL_8 - pulp_timestamp: 20241218T154614 pulp_repo_name: ohpc - '9': + pulp_timestamp: 20241218T154614 repo_file: OpenHPC + '9': pulp_path: OpenHPC/3/EL_9 - pulp_timestamp: 20241218T154614 pulp_repo_name: ohpc + pulp_timestamp: 20241218T154614 + repo_file: OpenHPC OpenHPC-updates: '8': - repo_file: OpenHPC pulp_path: OpenHPC/2/updates/EL_8 - pulp_timestamp: 20250512T003315 pulp_repo_name: ohpc-updates - '9': + pulp_timestamp: 20250512T003315 repo_file: OpenHPC + '9': pulp_path: OpenHPC/3/updates/EL_9 - pulp_timestamp: 20250510T003301 pulp_repo_name: ohpc-updates - epel: - epel: - '8': - repo_file: epel - pulp_path: epel/8/Everything/x86_64 - pulp_timestamp: 20250615T234151 - pulp_repo_name: epel - '9': - repo_file: epel - pulp_path: epel/9/Everything/x86_64 - pulp_timestamp: 20250729T235750 - pulp_repo_name: epel - -dnf_repos_extra: {} + pulp_timestamp: 20250510T003301 + repo_file: OpenHPC From 8f438b1a4f3de07819163b2d652f4c01ddde4e61 Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Mon, 18 Aug 2025 11:45:56 +0100 Subject: [PATCH 11/30] removed pulp_repo_name fields --- .../filter_plugins/pulp-list-filters.py | 3 ++- .../inventory/group_vars/all/timestamps.yml | 26 ------------------- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py index f903d5385..3859f310c 100644 --- a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py +++ b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py @@ -9,7 +9,7 @@ def filters(self): def select_repos(self, dnf_repos, target_distro_ver): #TODO: why does baseos get a major and minor version? """ Filter dnf_repos to only those for a relevant distribution version (M.m or M). Returns a list of dicts. - TODO: note this adds distro_ver as a key + TODO: note this adds distro_ver and pulp_repo_name as a key """ target_distro_ver_major = target_distro_ver.split('.')[0] @@ -25,6 +25,7 @@ def select_repos(self, dnf_repos, target_distro_ver): #TODO: why does baseos get raise ValueError(f'No key matching {target_distro_ver_major} or {target_distro_ver} found in f{repokey}') repo_data = dnf_repos[repokey][selected_ver] repo_data['distro_ver'] = selected_ver + repo_data['pulp_repo_name'] = repokey rpm_repos.append(repo_data) return rpm_repos diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml index 4589a9c2d..c8ff64c51 100644 --- a/environments/common/inventory/group_vars/all/timestamps.yml +++ b/environments/common/inventory/group_vars/all/timestamps.yml @@ -3,143 +3,117 @@ dnf_repos_default: Ceph: '8': pulp_path: centos/8-stream/storage/x86_64/ceph-quincy - pulp_repo_name: ceph pulp_timestamp: 20231104T015751 repo_file: ceph '9': pulp_path: centos/9-stream/storage/x86_64/ceph-reef - pulp_repo_name: ceph pulp_timestamp: 20250617T023108 repo_file: ceph appstream: '8.10': pulp_path: rocky/8.10/AppStream/x86_64/os - pulp_repo_name: appstream pulp_timestamp: 20250614T013846 repo_file: Rocky-AppStream '9.4': pulp_path: rocky/9.4/AppStream/x86_64/os - pulp_repo_name: appstream pulp_timestamp: 20241112T003151 repo_file: rocky '9.5': pulp_path: rocky/9.5/AppStream/x86_64/os - pulp_repo_name: appstream pulp_timestamp: 20250514T014704 repo_file: rocky '9.6': pulp_path: rocky/9.6/AppStream/x86_64/os - pulp_repo_name: appstream pulp_timestamp: 20250816T020215 repo_file: rocky baseos: '8.10': pulp_path: rocky/8.10/BaseOS/x86_64/os - pulp_repo_name: baseos pulp_timestamp: 20250614T013846 repo_file: Rocky-BaseOS '9.4': pulp_path: rocky/9.4/BaseOS/x86_64/os - pulp_repo_name: baseos pulp_timestamp: 20241115T011711 repo_file: rocky '9.5': pulp_path: rocky/9.5/BaseOS/x86_64/os - pulp_repo_name: baseos pulp_timestamp: 20250513T031844 repo_file: rocky '9.6': pulp_path: rocky/9.6/BaseOS/x86_64/os - pulp_repo_name: baseos pulp_timestamp: 20250815T050653 repo_file: rocky crb: '8.10': pulp_path: rocky/8.10/PowerTools/x86_64/os - pulp_repo_name: crb pulp_timestamp: 20250614T013846 repo_file: Rocky-PowerTools repo_name: powertools '9.4': pulp_path: rocky/9.4/CRB/x86_64/os - pulp_repo_name: crb pulp_timestamp: 20241115T003133 repo_file: rocky '9.5': pulp_path: rocky/9.5/CRB/x86_64/os - pulp_repo_name: crb pulp_timestamp: 20250514T014704 repo_file: rocky '9.6': pulp_path: rocky/9.6/CRB/x86_64/os - pulp_repo_name: crb pulp_timestamp: 20250815T034418 repo_file: rocky extras: '8.10': pulp_path: rocky/8.10/extras/x86_64/os - pulp_repo_name: extras pulp_timestamp: 20250510T032327 repo_file: Rocky-Extras '9.4': pulp_path: rocky/9.4/extras/x86_64/os - pulp_repo_name: extras pulp_timestamp: 20241118T002802 repo_file: rocky-extras '9.5': pulp_path: rocky/9.5/extras/x86_64/os - pulp_repo_name: extras pulp_timestamp: 20250506T032818 repo_file: rocky-extras '9.6': pulp_path: rocky/9.6/extras/x86_64/os - pulp_repo_name: extras pulp_timestamp: 20250726T040613 repo_file: rocky-extras grafana: '8': pulp_path: grafana/oss/rpm - pulp_repo_name: grafana pulp_timestamp: 20250730T011314 repo_file: grafana timestamp: 20250615T005738 '9': pulp_path: grafana/oss/rpm - pulp_repo_name: grafana pulp_timestamp: 20250730T011314 repo_file: grafana epel: epel: '8': pulp_path: epel/8/Everything/x86_64 - pulp_repo_name: epel pulp_timestamp: 20250615T234151 repo_file: epel '9': pulp_path: epel/9/Everything/x86_64 - pulp_repo_name: epel pulp_timestamp: 20250817T000753 repo_file: epel ohpc: OpenHPC: '8': pulp_path: OpenHPC/2/EL_8 - pulp_repo_name: ohpc pulp_timestamp: 20241218T154614 repo_file: OpenHPC '9': pulp_path: OpenHPC/3/EL_9 - pulp_repo_name: ohpc pulp_timestamp: 20241218T154614 repo_file: OpenHPC OpenHPC-updates: '8': pulp_path: OpenHPC/2/updates/EL_8 - pulp_repo_name: ohpc-updates pulp_timestamp: 20250512T003315 repo_file: OpenHPC '9': pulp_path: OpenHPC/3/updates/EL_9 - pulp_repo_name: ohpc-updates pulp_timestamp: 20250510T003301 repo_file: OpenHPC From d463feda746700d41465caad915826331ac57f02 Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Mon, 18 Aug 2025 14:25:39 +0100 Subject: [PATCH 12/30] updated docs, added gpg checks, simplified filters --- ansible/adhoc/deploy-pulp.yml | 2 +- .../roles/dnf_repos/tasks/disable_repos.yml | 2 +- ansible/roles/dnf_repos/tasks/set_repos.yml | 4 +-- ansible/roles/pulp_site/defaults/main.yml | 5 +++- .../filter_plugins/pulp-list-filters.py | 25 +++++++++++-------- docs/experimental/pulp.md | 23 +++++++++++------ .../common/files/grafana/grafana.repo.j2 | 2 +- .../inventory/group_vars/all/dnf_repos.yml | 14 ++++++++--- environments/common/inventory/groups | 4 +++ environments/site/inventory/groups | 6 +++++ 10 files changed, 60 insertions(+), 27 deletions(-) diff --git a/ansible/adhoc/deploy-pulp.yml b/ansible/adhoc/deploy-pulp.yml index 98d002d7a..f68259698 100644 --- a/ansible/adhoc/deploy-pulp.yml +++ b/ansible/adhoc/deploy-pulp.yml @@ -2,7 +2,7 @@ - name: Install pulp on server become: yes - hosts: pulp_server # TODO: add this to common/groups + hosts: pulp_server tasks: - name: Install pulp ansible.builtin.include_role: diff --git a/ansible/roles/dnf_repos/tasks/disable_repos.yml b/ansible/roles/dnf_repos/tasks/disable_repos.yml index 9f8abe6d9..313fa12bd 100644 --- a/ansible/roles/dnf_repos/tasks/disable_repos.yml +++ b/ansible/roles/dnf_repos/tasks/disable_repos.yml @@ -14,7 +14,7 @@ file: epel description: "{{ dnf_repos_epel_description }}" baseurl: "{{ dnf_repos_epel_baseurl }}" - gpgcheck: false + gpgcheck: true enabled: false - name: Get all repo files diff --git a/ansible/roles/dnf_repos/tasks/set_repos.yml b/ansible/roles/dnf_repos/tasks/set_repos.yml index d35bfe260..000aa70ff 100644 --- a/ansible/roles/dnf_repos/tasks/set_repos.yml +++ b/ansible/roles/dnf_repos/tasks/set_repos.yml @@ -8,7 +8,7 @@ description: "{{ repo_name }}" username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" - gpgcheck: false + gpgcheck: true loop: "{{ dnf_repos_no_epel | dict2items }}" loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" @@ -31,7 +31,7 @@ description: "{{ repo_name }}" username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" - gpgcheck: false # TODO: is this really false here and above?? + gpgcheck: true loop: "{{ dnf_repos_default_epel | dict2items }}" loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml index c2c269baa..7373a6030 100644 --- a/ansible/roles/pulp_site/defaults/main.yml +++ b/ansible/roles/pulp_site/defaults/main.yml @@ -1,7 +1,10 @@ pulp_site_url: "{{ appliances_pulp_url }}" pulp_site_port: 8080 pulp_site_username: admin # shouldn't be changed -pulp_site_password: "{{ vault_pulp_admin_password }}" #todo make more obvious this is different from the password needed for ark (pulp_site_upstream_password) +pulp_site_password: "{{ vault_pulp_admin_password }}" +# See environments/common/inventory/groups_vars/all/pulp.yml +# pulp_site_upstream_username: +# pulp_site_upstream_password: pulp_site_upstream_content_url: https://ark.stackhpc.com/pulp/content pulp_site_default_upstream_suffix: "{{ pulp_site_target_arch }}/os" pulp_site_validate_certs: false diff --git a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py index 3859f310c..84cb80b34 100644 --- a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py +++ b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py @@ -9,7 +9,8 @@ def filters(self): def select_repos(self, dnf_repos, target_distro_ver): #TODO: why does baseos get a major and minor version? """ Filter dnf_repos to only those for a relevant distribution version (M.m or M). Returns a list of dicts. - TODO: note this adds distro_ver and pulp_repo_name as a key + Also adds pulp_repo_name field to give the repository a unique name in Pulp to be referenced by subsequent + filters """ target_distro_ver_major = target_distro_ver.split('.')[0] @@ -24,35 +25,39 @@ def select_repos(self, dnf_repos, target_distro_ver): #TODO: why does baseos get else: raise ValueError(f'No key matching {target_distro_ver_major} or {target_distro_ver} found in f{repokey}') repo_data = dnf_repos[repokey][selected_ver] - repo_data['distro_ver'] = selected_ver - repo_data['pulp_repo_name'] = repokey + repo_data['pulp_repo_name'] = f"{repokey}-{selected_ver}-{dnf_repos[repokey][selected_ver]['pulp_timestamp']}" rpm_repos.append(repo_data) return rpm_repos def to_rpm_repos(self, rpm_info, content_url, repo_defaults): - """ TODO """ + """ Filter repo object list given by select_repos into dict required by the pulp_repository_rpm_repos variable + from stackhpc.pulp.pulp_repository role + """ rpm_repos = [] for repo_data in rpm_info: rpm_data = repo_defaults.copy() # NB: this changes behaviour vs before, so now defaults can correctly be overriden - rpm_data['name'] = get_repo_name(repo_data) + rpm_data['name'] = repo_data['pulp_repo_name'] rpm_data['url'] = '/'.join([content_url, repo_data['pulp_path'], repo_data['pulp_timestamp']]) rpm_data['state'] = 'present' rpm_repos.append(rpm_data) return rpm_repos def to_rpm_pubs(self, list): + """ Filter repo object list given by select_repos into dict required by the pulp_publication_rpm variable + from stackhpc.pulp.pulp_publication role + """ pub_list = map(lambda x: { - 'repository': get_repo_name(x), + 'repository': x['pulp_repo_name'], 'state': 'present' }, list) return pub_list def to_rpm_distros(self, list): + """ Filter repo object list given by select_repos into dict required by the pulp_distirubtion_rpm variable + from stackhpc.pulp.pulp_distribution role + """ distro_list = map(lambda x: { 'name': x['pulp_repo_name'], - 'repository': get_repo_name(x), + 'repository': x['pulp_repo_name'], 'base_path': '/'.join([x['pulp_path'],x['pulp_timestamp']]), 'state': 'present' }, list) return distro_list - -def get_repo_name(dnf_repos_data): - return f"{dnf_repos_data['pulp_repo_name']}-{dnf_repos_data['distro_ver']}-{dnf_repos_data['pulp_timestamp']}" diff --git a/docs/experimental/pulp.md b/docs/experimental/pulp.md index ee2580403..07db0dead 100644 --- a/docs/experimental/pulp.md +++ b/docs/experimental/pulp.md @@ -5,18 +5,17 @@ In order to ensure reproducible builds, the appliance can build images using rep ## Deploying/configuring Pulp Server ### Deploying a Pulp server -A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. To use this, create an inventory file defining a group `pulp_server` containing a single host. The hostvar `ansible_host` should be defined, giving the IP address Ansible should use for ssh. +A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. To use this, create an inventory file defining a group `pulp_server` containing a single host, which requires at least 2 vCPUs and 4GB RAM. Deploying and syncing Pulp has been tested on an RL9 host. The hostvar `ansible_host` should be defined, giving the IP address Ansible should use for ssh. For example: -**TODO: should be RL9 (or RL8?)** -**TODO: add size required (2 vCPUs, 4GB RAM)** -**TODO: example inventory file** +``` +[pulp_server] +pulp_host ansible_host= # Note the host name can't conflict with group names i.e can't be called `pulp` or `pulp_server` +``` Once complete, it will print a message giving a value to set for `appliances_pulp_url`, assuming the `ansible_host` address is also the address the cluster should use to reach the Pulp server. -**TODO: example config** - -Note access to this server's content isn't authenticated so this assumes the `pulp_server` host is not externall reachable. +Note access to this server's content isn't authenticated so this assumes the `pulp_server` host is not externally reachable. **TODO: You can actually do this using additional_nodes now, how would we make the pulp store persistant?** **TODO: don't advise that, we want single server for all environments** @@ -29,4 +28,12 @@ An existing Pulp server can be used to host Ark repos by overriding `pulp_site_p If the `pulp` group is added to the Packer build groups, the local Pulp server will be synced with Ark on build. You must authenticate with Ark by overriding `pulp_site_upstream_username` and `pulp_site_upstream_password` with your vault encrypted Ark dev credentials. `dnf_repos_username` and `dnf_repos_password` must remain unset to access content from the local Pulp. -Content can also be synced by running `ansible/adhoc/sync-pulp.yml`. By default this syncs repositories for Rocky 9.5 but this can be overridden by setting extra variables for `pulp_site_target_arch`, `pulp_site_target_distribution`, `pulp_site_target_distribution_version` and `pulp_site_target_distribution_version_major`. +Content can also be synced by running `ansible/adhoc/sync-pulp.yml`. By default this syncs repositories for the latest version of Rocky supported by the appliance but this can be overridden by setting extra variables for `pulp_site_target_arch`, `pulp_site_target_distribution` and `pulp_site_target_distribution_version`. + +## Example config in site variables + +``` +appliances_pulp_url: "http://:8080" +pulp_site_upstream_username: +pulp_site_upstream_password: +``` diff --git a/environments/common/files/grafana/grafana.repo.j2 b/environments/common/files/grafana/grafana.repo.j2 index 8f1aef5a7..fe4382a58 100644 --- a/environments/common/files/grafana/grafana.repo.j2 +++ b/environments/common/files/grafana/grafana.repo.j2 @@ -1,6 +1,6 @@ {{ ansible_managed | comment }} [grafana] -baseurl = {{ appliances_pulp_url }}/pulp/content/{{ appliances_pulp_repos.grafana[ansible_distribution_major_version] | appliances_repo_to_subpath }} +baseurl = {{ appliances_pulp_url }}/pulp/content/{{ dnf_repos_all['grafana'][ansible_distribution_version_major]['pulp_path'] }}/{{ dnf_repos_all['grafana'][ansible_distribution_major_version]['pulp_timestamp'] }} enabled = 0 name = grafana async = 1 diff --git a/environments/common/inventory/group_vars/all/dnf_repos.yml b/environments/common/inventory/group_vars/all/dnf_repos.yml index 05514e5b1..bbeeef295 100644 --- a/environments/common/inventory/group_vars/all/dnf_repos.yml +++ b/environments/common/inventory/group_vars/all/dnf_repos.yml @@ -1,10 +1,11 @@ -dnf_repos_all: "{{ dnf_repos_no_epel | combine(dnf_repos_default_epel) }}" #see timestamps.yml +dnf_repos_all: "{{ dnf_repos_no_epel | combine(dnf_repos_default_epel) }}" dnf_repos_no_epel: | {{ dnf_repos_default['base'] | combine(dnf_repos_default['ohpc'] if (openhpc_install_type | default('ohpc')) == 'ohpc' else {}) | combine(dnf_repos_extra) }} dnf_repos_default_epel: "{{ dnf_repos_default['epel'] }}" -dnf_repos_extra: {} + +# see timestamps.yml for dnf_repos_default definition, default repos should be in format # dnf_repos_default: # base: # top level keys for internal indexing only, see `dnf_repos_all` and `dnf_repos_no_epel` # appstream: # yum_repository:name @@ -14,4 +15,11 @@ dnf_repos_extra: {} # pulp_path: rocky/8.10/AppStream/x86_64/os # pulp_timestamp: 20250614T013846 # # pulp_content_url: # optional, dnf_repos_pulp_content_url -# pulp_repo_name: appstream # pulp repository name \ No newline at end of file +# pulp_repo_name: appstream # pulp repository name + +# Should be in same format as dnf_repos_default, except without the top level indexing keys e.g +# dnf_repos_extra: +# appstream: +# 8.10: +# ... +dnf_repos_extra: {} \ No newline at end of file diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups index 3b567df3a..bf734f7f4 100644 --- a/environments/common/inventory/groups +++ b/environments/common/inventory/groups @@ -213,4 +213,8 @@ extra_packages # Hosts to configure for node health checks - either entire 'compute' group or empty [pulp_server] +# Host to deploy a Pulp server on and sync with mirrors of upstream Ark repositories. Should be a group containing a single VM provisioned +# separately from the appliance. e.g +# pulp_host ansible_host= +# Note the host name can't conflict with group names i.e can't be called `pulp` or `pulp_server` diff --git a/environments/site/inventory/groups b/environments/site/inventory/groups index 9df61dc13..d30eb10a3 100644 --- a/environments/site/inventory/groups +++ b/environments/site/inventory/groups @@ -157,3 +157,9 @@ compute # Should be set to `compute` if enabled # Note that this feature currently assumes all compute nodes are VMs, enabling # when the cluster contains baremetal compute nodes may lead to unexpected scheduling behaviour + +[pulp_server] +# Host to deploy a Pulp server on and sync with mirrors of upstream Ark repositories. Should be a group containing a single VM provisioned +# separately from the appliance. e.g +# pulp_host ansible_host= +# Note the host name can't conflict with group names i.e can't be called `pulp` or `pulp_server` From 530ecf7ea96c8d3ea6d8bc216c478f976b9bb261 Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Mon, 18 Aug 2025 15:32:57 +0100 Subject: [PATCH 13/30] Added pulp systemd file + removed unused vars --- ansible/roles/pulp_site/defaults/main.yml | 2 -- ansible/roles/pulp_site/files/pulp.service | 12 ++++++++++++ ansible/roles/pulp_site/tasks/install.yml | 14 ++++++++++++++ docs/experimental/pulp.md | 1 - 4 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 ansible/roles/pulp_site/files/pulp.service diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml index 7373a6030..205d29ee1 100644 --- a/ansible/roles/pulp_site/defaults/main.yml +++ b/ansible/roles/pulp_site/defaults/main.yml @@ -6,8 +6,6 @@ pulp_site_password: "{{ vault_pulp_admin_password }}" # pulp_site_upstream_username: # pulp_site_upstream_password: pulp_site_upstream_content_url: https://ark.stackhpc.com/pulp/content -pulp_site_default_upstream_suffix: "{{ pulp_site_target_arch }}/os" -pulp_site_validate_certs: false pulp_site_install_dir: '/home/rocky/pulp' pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}" pulp_site_target_facts: "{{ hostvars[groups['pulp'][0]]['ansible_facts'] }}" diff --git a/ansible/roles/pulp_site/files/pulp.service b/ansible/roles/pulp_site/files/pulp.service new file mode 100644 index 000000000..464961d23 --- /dev/null +++ b/ansible/roles/pulp_site/files/pulp.service @@ -0,0 +1,12 @@ +# Adapted from https://grimoire.carcano.ch/blog/installing-pulp3-as-a-container/ +[Unit] +Description=Pulp +Wants=syslog.service + +[Service] +Restart=always +ExecStart=/usr/bin/podman start -a pulp +ExecStop=/usr/bin/podman stop -t 15 pulp + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/pulp_site/tasks/install.yml b/ansible/roles/pulp_site/tasks/install.yml index 39b4fcd97..46ad5213e 100644 --- a/ansible/roles/pulp_site/tasks/install.yml +++ b/ansible/roles/pulp_site/tasks/install.yml @@ -32,7 +32,21 @@ - "{{ pulp_site_install_dir }}/containers:/var/lib/containers{{ pulp_site_selinux_suffix }}" device: /dev/fuse image: docker.io/pulp/pulp:3.68.1 + state: present +- name: Create systemd file + copy: + src: pulp.service + dest: /etc/systemd/system/pulp.service + register: _pulp_service + +- name: Start Pulp service + systemd: + name: pulp + state: "{{ 'started' if _pulp_service.changed else 'restarted' }}" + daemon_reload: "{{ _pulp_service.changed }}" + enabled: true + - name: Reset admin password once container has initialised no_log: true ansible.builtin.shell: diff --git a/docs/experimental/pulp.md b/docs/experimental/pulp.md index 07db0dead..702003dcf 100644 --- a/docs/experimental/pulp.md +++ b/docs/experimental/pulp.md @@ -19,7 +19,6 @@ Note access to this server's content isn't authenticated so this assumes the `pu **TODO: You can actually do this using additional_nodes now, how would we make the pulp store persistant?** **TODO: don't advise that, we want single server for all environments** -**TODO: Add a systemd unit to run pulp!** ### Using an existing Pulp server An existing Pulp server can be used to host Ark repos by overriding `pulp_site_password` and `appliances_pulp_url` in the target environment. Note that this assumes the same configuration as the appliance deployed Pulp i.e no content authentication. From 7a0eab3f369620a2f5cdcb2d8fef0815d161ead3 Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Tue, 19 Aug 2025 10:43:14 +0100 Subject: [PATCH 14/30] added READMEs + updated variable names --- ansible/roles/dnf_repos/README.md | 46 +++++++++++++++++++ ansible/roles/pulp_site/README.md | 35 ++++++++++++++ ansible/roles/pulp_site/defaults/main.yml | 10 ++-- ansible/roles/pulp_site/tasks/install.yml | 8 ++-- .../inventory/group_vars/all/dnf_repos.yml | 1 - 5 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 ansible/roles/dnf_repos/README.md create mode 100644 ansible/roles/pulp_site/README.md diff --git a/ansible/roles/dnf_repos/README.md b/ansible/roles/dnf_repos/README.md new file mode 100644 index 000000000..c98949a87 --- /dev/null +++ b/ansible/roles/dnf_repos/README.md @@ -0,0 +1,46 @@ +dnf_repos +========= + +Modifies repo definitions for repofiles in `/etc/yum.repos.d` to point to snapshots in StackHPC's Ark Pulp server. + +Requirements +------------ + +Requires Ark credentials. + +Role Variables +-------------- + +Variables in this role are also required by `pulp_site` so set in +`environments/common/inventory/groups_vars/all/dnf_repos.yml`. See that file for detailed default values. + +- `dnf_repos_all`: Dict of dicts containing information to construct URLs for timestamped repos from Ark for each Rocky version. For example: + ``` + dnf_repos_all: + appstream: # yum_repository:name + '8.10': # ansible_distribution_version or ansible_distribution_major_version + repo_file: Rocky-AppStream # yum_repository: file + # repo_name: # optional, override yum_repository:name + pulp_path: rocky/8.10/AppStream/x86_64/os + pulp_timestamp: 20250614T013846 + # pulp_content_url: # optional, dnf_repos_pulp_content_url + '9.6': + ... + ``` +- `dnf_repos_default`: Appliance default repos to use Ark snapshots for. Follows same format as + `dnf_repos_all`, but includes top level keys to allow repos to be conditionally included in + `dnf_repos_all`. See `environments/common/inventory/group_vars/all/dnf_repos.yml` and + `environments/common/inventory/group_vars/all/timestamps.yml` for full templating logic. +- `dnf_repos_extra`: Additional repos to use Ark snapshots for. Follows same format as + `dnf_repos_all`. Defaults to `{}` +- `dnf_repos_no_epel`: Dict of all repos included in `dnf_repos_all` excluding + `epel`, used to prevent conflicts with repofile installed by `epel-release` +- `dnf_repos_default_epel`: Dict of repos objects following same format as `dnf_repos_all` but only + including `epel` repo. +- `dnf_repos_pulp_content_url`: Optional str. Content URL of Pulp server to use Ark snapshots from. + Should be overriden if using local Pulp server instead of upstream Ark Pulp (See + `ansible/roles/pulp_site`). Defaults to `{{ appliances_pulp_url }}/pulp/content` +- `dnf_repos_username`: Optional str. Username for Ark. Should be set if using upstream StackHPC Ark + Pulp server, but omitted if using local Pulp server (see `ansible/roles/pulp_site`) +- `dnf_repos_password`: Optional str. Password for Ark. Should be set if using upstream StackHPC Ark + Pulp server, but omitted if using local Pulp server (see `ansible/roles/pulp_site`) diff --git a/ansible/roles/pulp_site/README.md b/ansible/roles/pulp_site/README.md new file mode 100644 index 000000000..086053fab --- /dev/null +++ b/ansible/roles/pulp_site/README.md @@ -0,0 +1,35 @@ +pulp_site +========= + +Contains playbooks to deploy a Pulp server and sync its content with repo snapshots in +StackHPC's Ark Pulp server + +Requirements +------------ + +Requires Ark credentials. The VM you are deploying Pulp on must allow ingress on `pulp_site_port` +and not be externally accessible (as the Pulp server's content is unauthenticated). + +Role Variables +-------------- + +- `pulp_site_url`: Required str. The base url from which Pulp content will be hosted. Defaults to `{{ appliances_pulp_url }}`. + Value to set for ``appliances_pulp_url` will be generated and output by the deploy.yml playbook. +- `pulp_site_port`: Optional str. Port to serve Pulp server on. Defaults to `8080`. +- `pulp_site_username`: Optional str. Admin username for the Pulp server. Defaults to `admin`. +- `pulp_site_password`: Required str. Admin password for the Pulp server. Defaults to `{{ vault_pulp_admin_password }}`. +- `pulp_site_upstream_username`: Required str. Username for accessing content from the upstream Ark Pulp server. +- `pulp_site_upstream_password`: Required str. Password for upstream Ark Pulp server. +- `pulp_site_upstream_content_url`: Optional str. Content URL of upstream Ark Pulp. Defaults to `https://ark.stackhpc.com/pulp/content`. +- `pulp_site_install_dir`: Optional str. Directory on Pulp host to install config and persistent state to be mounted into Pulp container. Defaults to `/home/rocky/pulp`. +- `pulp_site_target_facts`: Optional str. The `ansible_facts` of a host which will be pulling from your Pulp server, allowing the role to auto-discover the necessary repos to pull. + defaults to `{{ hostvars[groups['pulp'][0]]['ansible_facts'] }}`. +- `pulp_site_target_distribution_version`: Optional str. The Rocky Linux minor release to sync repos from Ark for. Defaults to `{{ pulp_site_target_facts['distribution_version'] }}`. +- `pulp_site_rpm_repo_defaults`: Optional dict. Contains key value pairs for fields which are common to all repo definition in `pulp_site_rpm_repos`. Includes values for `remote_username`, + `remote_password` and `policy` by default. +- `pulp_site_rpm_repos`: Optional list of dicts. List of repo definitions in format required by the `stackhpc.pulp.pulp_repository`. Defaults to modified versions of repos defined in + `dnf_repos_all`. +- `pulp_site_rpm_publications`: Optional list of dicts. List of repo definitions in format required by the `stackhpc.pulp.pulp_publication`. Defaults to list of publications for repos defined in + `dnf_repos_all`. +- `pulp_site_rpm_distributions`: Optional list of dicts. List of repo definitions in format required by the `stackhpc.pulp.pulp_distribution`. Defaults to list of distributions for repos defined in + `dnf_repos_all`. diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml index 205d29ee1..1eedf907c 100644 --- a/ansible/roles/pulp_site/defaults/main.yml +++ b/ansible/roles/pulp_site/defaults/main.yml @@ -7,7 +7,7 @@ pulp_site_password: "{{ vault_pulp_admin_password }}" # pulp_site_upstream_password: pulp_site_upstream_content_url: https://ark.stackhpc.com/pulp/content pulp_site_install_dir: '/home/rocky/pulp' -pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}" +_pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}" pulp_site_target_facts: "{{ hostvars[groups['pulp'][0]]['ansible_facts'] }}" pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}" # TODO: how to set automatically? @@ -16,16 +16,16 @@ pulp_site_rpm_repo_defaults: remote_password: "{{ pulp_site_upstream_password }}" policy: on_demand -pulp_site_rpm_info: | +_pulp_site_rpm_info: | {{ dnf_repos_all | select_repos(pulp_site_target_distribution_version) }} pulp_site_rpm_repos: | {{ - pulp_site_rpm_info | + _pulp_site_rpm_info | to_rpm_repos(pulp_site_upstream_content_url, pulp_site_rpm_repo_defaults) }} -pulp_site_rpm_publications: "{{ pulp_site_rpm_info | to_rpm_pubs }}" -pulp_site_rpm_distributions: "{{ pulp_site_rpm_info | to_rpm_distros }}" +pulp_site_rpm_publications: "{{ _pulp_site_rpm_info | to_rpm_pubs }}" +pulp_site_rpm_distributions: "{{ _pulp_site_rpm_info | to_rpm_distros }}" diff --git a/ansible/roles/pulp_site/tasks/install.yml b/ansible/roles/pulp_site/tasks/install.yml index 46ad5213e..75b0f66cb 100644 --- a/ansible/roles/pulp_site/tasks/install.yml +++ b/ansible/roles/pulp_site/tasks/install.yml @@ -26,10 +26,10 @@ publish: - "{{ pulp_site_port }}:80" volume: - - "{{ pulp_site_install_dir }}/settings:/etc/pulp{{ pulp_site_selinux_suffix }}" - - "{{ pulp_site_install_dir }}/pulp_storage:/var/lib/pulp{{ pulp_site_selinux_suffix }}" - - "{{ pulp_site_install_dir }}/pgsql:/var/lib/pgsql{{ pulp_site_selinux_suffix }}" - - "{{ pulp_site_install_dir }}/containers:/var/lib/containers{{ pulp_site_selinux_suffix }}" + - "{{ pulp_site_install_dir }}/settings:/etc/pulp{{ _pulp_site_selinux_suffix }}" + - "{{ pulp_site_install_dir }}/pulp_storage:/var/lib/pulp{{ _pulp_site_selinux_suffix }}" + - "{{ pulp_site_install_dir }}/pgsql:/var/lib/pgsql{{ _pulp_site_selinux_suffix }}" + - "{{ pulp_site_install_dir }}/containers:/var/lib/containers{{ _pulp_site_selinux_suffix }}" device: /dev/fuse image: docker.io/pulp/pulp:3.68.1 state: present diff --git a/environments/common/inventory/group_vars/all/dnf_repos.yml b/environments/common/inventory/group_vars/all/dnf_repos.yml index bbeeef295..bd1029872 100644 --- a/environments/common/inventory/group_vars/all/dnf_repos.yml +++ b/environments/common/inventory/group_vars/all/dnf_repos.yml @@ -15,7 +15,6 @@ dnf_repos_default_epel: "{{ dnf_repos_default['epel'] }}" # pulp_path: rocky/8.10/AppStream/x86_64/os # pulp_timestamp: 20250614T013846 # # pulp_content_url: # optional, dnf_repos_pulp_content_url -# pulp_repo_name: appstream # pulp repository name # Should be in same format as dnf_repos_default, except without the top level indexing keys e.g # dnf_repos_extra: From 305a057d3d738f7c200103e3adb09494a596d52e Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Tue, 19 Aug 2025 10:45:25 +0100 Subject: [PATCH 15/30] disabled gpg checks for dnf_repos --- ansible/roles/dnf_repos/tasks/disable_repos.yml | 2 +- ansible/roles/dnf_repos/tasks/set_repos.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/dnf_repos/tasks/disable_repos.yml b/ansible/roles/dnf_repos/tasks/disable_repos.yml index 313fa12bd..9f8abe6d9 100644 --- a/ansible/roles/dnf_repos/tasks/disable_repos.yml +++ b/ansible/roles/dnf_repos/tasks/disable_repos.yml @@ -14,7 +14,7 @@ file: epel description: "{{ dnf_repos_epel_description }}" baseurl: "{{ dnf_repos_epel_baseurl }}" - gpgcheck: true + gpgcheck: false enabled: false - name: Get all repo files diff --git a/ansible/roles/dnf_repos/tasks/set_repos.yml b/ansible/roles/dnf_repos/tasks/set_repos.yml index 000aa70ff..f5d19207f 100644 --- a/ansible/roles/dnf_repos/tasks/set_repos.yml +++ b/ansible/roles/dnf_repos/tasks/set_repos.yml @@ -8,7 +8,7 @@ description: "{{ repo_name }}" username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" - gpgcheck: true + gpgcheck: false loop: "{{ dnf_repos_no_epel | dict2items }}" loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" @@ -31,7 +31,7 @@ description: "{{ repo_name }}" username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" - gpgcheck: true + gpgcheck: false loop: "{{ dnf_repos_default_epel | dict2items }}" loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" From 1b4c867f90693d0d075b0deb1eb67749d1fc0edc Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Tue, 19 Aug 2025 11:40:11 +0100 Subject: [PATCH 16/30] typo --- environments/common/files/grafana/grafana.repo.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/common/files/grafana/grafana.repo.j2 b/environments/common/files/grafana/grafana.repo.j2 index fe4382a58..cc736edf2 100644 --- a/environments/common/files/grafana/grafana.repo.j2 +++ b/environments/common/files/grafana/grafana.repo.j2 @@ -1,6 +1,6 @@ {{ ansible_managed | comment }} [grafana] -baseurl = {{ appliances_pulp_url }}/pulp/content/{{ dnf_repos_all['grafana'][ansible_distribution_version_major]['pulp_path'] }}/{{ dnf_repos_all['grafana'][ansible_distribution_major_version]['pulp_timestamp'] }} +baseurl = {{ appliances_pulp_url }}/pulp/content/{{ dnf_repos_all['grafana'][ansible_distribution_major_version]['pulp_path'] }}/{{ dnf_repos_all['grafana'][ansible_distribution_major_version]['pulp_timestamp'] }} enabled = 0 name = grafana async = 1 From 767addd8d7f8c5314ed5428addf321c1d520022a Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Tue, 19 Aug 2025 13:06:21 +0100 Subject: [PATCH 17/30] fixed disable repos task --- .../roles/dnf_repos/tasks/disable_repos.yml | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/ansible/roles/dnf_repos/tasks/disable_repos.yml b/ansible/roles/dnf_repos/tasks/disable_repos.yml index 9f8abe6d9..61459e547 100644 --- a/ansible/roles/dnf_repos/tasks/disable_repos.yml +++ b/ansible/roles/dnf_repos/tasks/disable_repos.yml @@ -1,21 +1,20 @@ --- - name: Remove password and disable Pulp repos ansible.builtin.yum_repository: - file: "{{ item.file }}" - name: "{{ item.name }}" - baseurl: "{{ item.base_url }}" - description: "{{ item.name }}" + file: "{{ repo_values.repo_file }}" + name: "{{ repo_name }}" + baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}" + description: "{{ repo_name }}" enabled: false - loop: "{{ dnf_repos_repolist }}" - -- name: Remove password and disable EPEL repo - ansible.builtin.yum_repository: - name: epel - file: epel - description: "{{ dnf_repos_epel_description }}" - baseurl: "{{ dnf_repos_epel_baseurl }}" gpgcheck: false - enabled: false + loop: "{{ dnf_repos_all | dict2items }}" + loop_control: + label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" + vars: + repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}" + repo_values: "{{ item.value[repo_os] }}" + repo_name: "{{ repo_values.repo_name | default(item.key) }}" + repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}" - name: Get all repo files ansible.builtin.find: From 8ad6869736cbdc9253e40253d10d857f1b7d9a35 Mon Sep 17 00:00:00 2001 From: wtripp180901 <78219569+wtripp180901@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:06:55 +0100 Subject: [PATCH 18/30] bump images --- environments/.stackhpc/tofu/cluster_image.auto.tfvars.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json b/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json index af148393a..88cdb4264 100644 --- a/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json +++ b/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json @@ -1,6 +1,6 @@ { "cluster_image": { - "RL8": "openhpc-RL8-250808-1727-faa44755", - "RL9": "openhpc-RL9-250808-1727-faa44755" + "RL8": "openhpc-RL8-250820-0800-767addd8", + "RL9": "openhpc-RL9-250820-0800-767addd8" } } From b5b3b393f002adee47f9caaebc154a50f1297528 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 27 Aug 2025 13:18:00 +0000 Subject: [PATCH 19/30] remove dnf_repos extra index/key and make epel/openhpc special-cases simpler --- ansible/roles/dnf_repos/defaults/main.yml | 6 +- .../roles/dnf_repos/tasks/disable_repos.yml | 2 +- ansible/roles/dnf_repos/tasks/set_repos.yml | 8 +- .../inventory/group_vars/all/dnf_repos.yml | 28 +-- .../inventory/group_vars/all/timestamps.yml | 231 +++++++++--------- 5 files changed, 127 insertions(+), 148 deletions(-) diff --git a/ansible/roles/dnf_repos/defaults/main.yml b/ansible/roles/dnf_repos/defaults/main.yml index aa9757e57..fe3c44e4f 100644 --- a/ansible/roles/dnf_repos/defaults/main.yml +++ b/ansible/roles/dnf_repos/defaults/main.yml @@ -1,8 +1,4 @@ -dnf_repos_default: {} # see environments/common/inventory/group_vars/all/timestamps.yml -dnf_repos_extra: {} -dnf_repos_no_epel: {} -dnf_repos_all: {} -dnf_repos_default_epel: {} +dnf_repos_repos: {} # see environments/common/inventory/group_vars/all/{dnf_repos,timestamps}.yml dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content" dnf_repos_username: "{{ omit }}" dnf_repos_password: "{{ omit }}" diff --git a/ansible/roles/dnf_repos/tasks/disable_repos.yml b/ansible/roles/dnf_repos/tasks/disable_repos.yml index 61459e547..4db073bc1 100644 --- a/ansible/roles/dnf_repos/tasks/disable_repos.yml +++ b/ansible/roles/dnf_repos/tasks/disable_repos.yml @@ -7,7 +7,7 @@ description: "{{ repo_name }}" enabled: false gpgcheck: false - loop: "{{ dnf_repos_all | dict2items }}" + loop: "{{ dnf_repos_repos | dict2items }}" loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" vars: diff --git a/ansible/roles/dnf_repos/tasks/set_repos.yml b/ansible/roles/dnf_repos/tasks/set_repos.yml index f5d19207f..2db4de91b 100644 --- a/ansible/roles/dnf_repos/tasks/set_repos.yml +++ b/ansible/roles/dnf_repos/tasks/set_repos.yml @@ -1,6 +1,6 @@ --- -- name: Replace system repos with Pulp repos +- name: Replace non-epel repos with Pulp repos ansible.builtin.yum_repository: file: "{{ repo_values.repo_file }}" name: "{{ repo_name }}" @@ -9,9 +9,10 @@ username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" gpgcheck: false - loop: "{{ dnf_repos_no_epel | dict2items }}" + loop: "{{ dnf_repos_repos | dict2items }}" loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" + when: repo_name != 'epel' vars: repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}" repo_values: "{{ item.value[repo_os] }}" @@ -32,9 +33,10 @@ username: "{{ dnf_repos_username }}" password: "{{ dnf_repos_password }}" gpgcheck: false - loop: "{{ dnf_repos_default_epel | dict2items }}" + loop: "{{ dnf_repos_repos | dict2items }}" loop_control: label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}" + when: repo_name == 'epel' vars: repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}" repo_values: "{{ item.value[repo_os] }}" diff --git a/environments/common/inventory/group_vars/all/dnf_repos.yml b/environments/common/inventory/group_vars/all/dnf_repos.yml index bd1029872..d920ed0de 100644 --- a/environments/common/inventory/group_vars/all/dnf_repos.yml +++ b/environments/common/inventory/group_vars/all/dnf_repos.yml @@ -1,24 +1,8 @@ -dnf_repos_all: "{{ dnf_repos_no_epel | combine(dnf_repos_default_epel) }}" -dnf_repos_no_epel: | - {{ dnf_repos_default['base'] - | combine(dnf_repos_default['ohpc'] if (openhpc_install_type | default('ohpc')) == 'ohpc' else {}) - | combine(dnf_repos_extra) }} -dnf_repos_default_epel: "{{ dnf_repos_default['epel'] }}" +# dnf_repos_default: see timestamps.yml -# see timestamps.yml for dnf_repos_default definition, default repos should be in format -# dnf_repos_default: -# base: # top level keys for internal indexing only, see `dnf_repos_all` and `dnf_repos_no_epel` -# appstream: # yum_repository:name -# '8.10': # ansible_distribution_version or ansible_distribution_major_version -# repo_file: Rocky-AppStream # yum_repository: file -# # repo_name: # optional, override yum_repository:name -# pulp_path: rocky/8.10/AppStream/x86_64/os -# pulp_timestamp: 20250614T013846 -# # pulp_content_url: # optional, dnf_repos_pulp_content_url +# override this in environments/site/inventory/group_vars/dnf_repos.yml to add repos: +dnf_repos_extra: {} -# Should be in same format as dnf_repos_default, except without the top level indexing keys e.g -# dnf_repos_extra: -# appstream: -# 8.10: -# ... -dnf_repos_extra: {} \ No newline at end of file +# indirection to skip openhpc repos if using alternative slurm: +dnf_repos_skip: "{{ [] if ((openhpc_install_type | default('ohpc') == 'ohpc')) else ['OpenHPC', 'OpenHPC-updates'] }}" +dnf_repos_repos: "{{ dnf_repos_default | combine(dnf_repos_extra) | dict2items | rejectattr('key', 'in', dnf_repos_skip) | items2dict }}" diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml index c8ff64c51..d2df041b4 100644 --- a/environments/common/inventory/group_vars/all/timestamps.yml +++ b/environments/common/inventory/group_vars/all/timestamps.yml @@ -1,119 +1,116 @@ dnf_repos_default: - base: - Ceph: - '8': - pulp_path: centos/8-stream/storage/x86_64/ceph-quincy - pulp_timestamp: 20231104T015751 - repo_file: ceph - '9': - pulp_path: centos/9-stream/storage/x86_64/ceph-reef - pulp_timestamp: 20250617T023108 - repo_file: ceph - appstream: - '8.10': - pulp_path: rocky/8.10/AppStream/x86_64/os - pulp_timestamp: 20250614T013846 - repo_file: Rocky-AppStream - '9.4': - pulp_path: rocky/9.4/AppStream/x86_64/os - pulp_timestamp: 20241112T003151 - repo_file: rocky - '9.5': - pulp_path: rocky/9.5/AppStream/x86_64/os - pulp_timestamp: 20250514T014704 - repo_file: rocky - '9.6': - pulp_path: rocky/9.6/AppStream/x86_64/os - pulp_timestamp: 20250816T020215 - repo_file: rocky - baseos: - '8.10': - pulp_path: rocky/8.10/BaseOS/x86_64/os - pulp_timestamp: 20250614T013846 - repo_file: Rocky-BaseOS - '9.4': - pulp_path: rocky/9.4/BaseOS/x86_64/os - pulp_timestamp: 20241115T011711 - repo_file: rocky - '9.5': - pulp_path: rocky/9.5/BaseOS/x86_64/os - pulp_timestamp: 20250513T031844 - repo_file: rocky - '9.6': - pulp_path: rocky/9.6/BaseOS/x86_64/os - pulp_timestamp: 20250815T050653 - repo_file: rocky - crb: - '8.10': - pulp_path: rocky/8.10/PowerTools/x86_64/os - pulp_timestamp: 20250614T013846 - repo_file: Rocky-PowerTools - repo_name: powertools - '9.4': - pulp_path: rocky/9.4/CRB/x86_64/os - pulp_timestamp: 20241115T003133 - repo_file: rocky - '9.5': - pulp_path: rocky/9.5/CRB/x86_64/os - pulp_timestamp: 20250514T014704 - repo_file: rocky - '9.6': - pulp_path: rocky/9.6/CRB/x86_64/os - pulp_timestamp: 20250815T034418 - repo_file: rocky - extras: - '8.10': - pulp_path: rocky/8.10/extras/x86_64/os - pulp_timestamp: 20250510T032327 - repo_file: Rocky-Extras - '9.4': - pulp_path: rocky/9.4/extras/x86_64/os - pulp_timestamp: 20241118T002802 - repo_file: rocky-extras - '9.5': - pulp_path: rocky/9.5/extras/x86_64/os - pulp_timestamp: 20250506T032818 - repo_file: rocky-extras - '9.6': - pulp_path: rocky/9.6/extras/x86_64/os - pulp_timestamp: 20250726T040613 - repo_file: rocky-extras - grafana: - '8': - pulp_path: grafana/oss/rpm - pulp_timestamp: 20250730T011314 - repo_file: grafana - timestamp: 20250615T005738 - '9': - pulp_path: grafana/oss/rpm - pulp_timestamp: 20250730T011314 - repo_file: grafana + Ceph: + '8': + pulp_path: centos/8-stream/storage/x86_64/ceph-quincy + pulp_timestamp: 20231104T015751 + repo_file: ceph + '9': + pulp_path: centos/9-stream/storage/x86_64/ceph-reef + pulp_timestamp: 20250617T023108 + repo_file: ceph + appstream: + '8.10': + pulp_path: rocky/8.10/AppStream/x86_64/os + pulp_timestamp: 20250614T013846 + repo_file: Rocky-AppStream + '9.4': + pulp_path: rocky/9.4/AppStream/x86_64/os + pulp_timestamp: 20241112T003151 + repo_file: rocky + '9.5': + pulp_path: rocky/9.5/AppStream/x86_64/os + pulp_timestamp: 20250514T014704 + repo_file: rocky + '9.6': + pulp_path: rocky/9.6/AppStream/x86_64/os + pulp_timestamp: 20250816T020215 + repo_file: rocky + baseos: + '8.10': + pulp_path: rocky/8.10/BaseOS/x86_64/os + pulp_timestamp: 20250614T013846 + repo_file: Rocky-BaseOS + '9.4': + pulp_path: rocky/9.4/BaseOS/x86_64/os + pulp_timestamp: 20241115T011711 + repo_file: rocky + '9.5': + pulp_path: rocky/9.5/BaseOS/x86_64/os + pulp_timestamp: 20250513T031844 + repo_file: rocky + '9.6': + pulp_path: rocky/9.6/BaseOS/x86_64/os + pulp_timestamp: 20250815T050653 + repo_file: rocky + crb: + '8.10': + pulp_path: rocky/8.10/PowerTools/x86_64/os + pulp_timestamp: 20250614T013846 + repo_file: Rocky-PowerTools + repo_name: powertools + '9.4': + pulp_path: rocky/9.4/CRB/x86_64/os + pulp_timestamp: 20241115T003133 + repo_file: rocky + '9.5': + pulp_path: rocky/9.5/CRB/x86_64/os + pulp_timestamp: 20250514T014704 + repo_file: rocky + '9.6': + pulp_path: rocky/9.6/CRB/x86_64/os + pulp_timestamp: 20250815T034418 + repo_file: rocky + extras: + '8.10': + pulp_path: rocky/8.10/extras/x86_64/os + pulp_timestamp: 20250510T032327 + repo_file: Rocky-Extras + '9.4': + pulp_path: rocky/9.4/extras/x86_64/os + pulp_timestamp: 20241118T002802 + repo_file: rocky-extras + '9.5': + pulp_path: rocky/9.5/extras/x86_64/os + pulp_timestamp: 20250506T032818 + repo_file: rocky-extras + '9.6': + pulp_path: rocky/9.6/extras/x86_64/os + pulp_timestamp: 20250726T040613 + repo_file: rocky-extras + grafana: + '8': + pulp_path: grafana/oss/rpm + pulp_timestamp: 20250730T011314 + repo_file: grafana + timestamp: 20250615T005738 + '9': + pulp_path: grafana/oss/rpm + pulp_timestamp: 20250730T011314 + repo_file: grafana epel: - epel: - '8': - pulp_path: epel/8/Everything/x86_64 - pulp_timestamp: 20250615T234151 - repo_file: epel - '9': - pulp_path: epel/9/Everything/x86_64 - pulp_timestamp: 20250817T000753 - repo_file: epel - ohpc: - OpenHPC: - '8': - pulp_path: OpenHPC/2/EL_8 - pulp_timestamp: 20241218T154614 - repo_file: OpenHPC - '9': - pulp_path: OpenHPC/3/EL_9 - pulp_timestamp: 20241218T154614 - repo_file: OpenHPC - OpenHPC-updates: - '8': - pulp_path: OpenHPC/2/updates/EL_8 - pulp_timestamp: 20250512T003315 - repo_file: OpenHPC - '9': - pulp_path: OpenHPC/3/updates/EL_9 - pulp_timestamp: 20250510T003301 - repo_file: OpenHPC + '8': + pulp_path: epel/8/Everything/x86_64 + pulp_timestamp: 20250615T234151 + repo_file: epel + '9': + pulp_path: epel/9/Everything/x86_64 + pulp_timestamp: 20250817T000753 + repo_file: epel + OpenHPC: + '8': + pulp_path: OpenHPC/2/EL_8 + pulp_timestamp: 20241218T154614 + repo_file: OpenHPC + '9': + pulp_path: OpenHPC/3/EL_9 + pulp_timestamp: 20241218T154614 + repo_file: OpenHPC + OpenHPC-updates: + '8': + pulp_path: OpenHPC/2/updates/EL_8 + pulp_timestamp: 20250512T003315 + repo_file: OpenHPC + '9': + pulp_path: OpenHPC/3/updates/EL_9 + pulp_timestamp: 20250510T003301 + repo_file: OpenHPC From 244a9ee5688f0670df5ccf50f20d64525d095780 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 27 Aug 2025 13:22:59 +0000 Subject: [PATCH 20/30] clarify pulp distro selection --- ansible/adhoc/sync-pulp.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/adhoc/sync-pulp.yml b/ansible/adhoc/sync-pulp.yml index c879d438b..373f3abb8 100644 --- a/ansible/adhoc/sync-pulp.yml +++ b/ansible/adhoc/sync-pulp.yml @@ -6,4 +6,5 @@ vars: pulp_site_target_arch: "x86_64" pulp_site_target_distribution: "rocky" - pulp_site_target_distribution_version: "{{ dnf_repos_all['baseos'].keys() | map('float') | sort | last }}" \ No newline at end of file + # default distribution to *latest* specified for baseos repo: + pulp_site_target_distribution_version: "{{ dnf_repos_repos['baseos'].keys() | map('float') | sort | last }}" From 32278e7e99a5fc7b7fb782828b6b70653ed83b65 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 27 Aug 2025 13:48:07 +0000 Subject: [PATCH 21/30] fixup sync vars --- ansible/roles/pulp_site/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/pulp_site/defaults/main.yml b/ansible/roles/pulp_site/defaults/main.yml index 1eedf907c..3d2bce7ec 100644 --- a/ansible/roles/pulp_site/defaults/main.yml +++ b/ansible/roles/pulp_site/defaults/main.yml @@ -18,7 +18,7 @@ pulp_site_rpm_repo_defaults: _pulp_site_rpm_info: | {{ - dnf_repos_all | + dnf_repos_repos | select_repos(pulp_site_target_distribution_version) }} pulp_site_rpm_repos: | From a63fa1fd8f4aa7489d68fe7f93ea1131ecb04ce8 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 27 Aug 2025 13:48:25 +0000 Subject: [PATCH 22/30] fixup grafana vars --- environments/common/files/grafana/grafana.repo.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/common/files/grafana/grafana.repo.j2 b/environments/common/files/grafana/grafana.repo.j2 index cc736edf2..6ce2581f6 100644 --- a/environments/common/files/grafana/grafana.repo.j2 +++ b/environments/common/files/grafana/grafana.repo.j2 @@ -1,6 +1,6 @@ {{ ansible_managed | comment }} [grafana] -baseurl = {{ appliances_pulp_url }}/pulp/content/{{ dnf_repos_all['grafana'][ansible_distribution_major_version]['pulp_path'] }}/{{ dnf_repos_all['grafana'][ansible_distribution_major_version]['pulp_timestamp'] }} +baseurl = {{ appliances_pulp_url }}/pulp/content/{{ dnf_repos_repos['grafana'][ansible_distribution_major_version]['pulp_path'] }}/{{ dnf_repos_repos['grafana'][ansible_distribution_major_version]['pulp_timestamp'] }} enabled = 0 name = grafana async = 1 From 4580c2ec9a884022119cd28af6869c7cffd2d80a Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Wed, 27 Aug 2025 14:02:00 +0000 Subject: [PATCH 23/30] revert latest timestamp changes for extra key level --- ansible/library/latest_timestamps.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/ansible/library/latest_timestamps.py b/ansible/library/latest_timestamps.py index fc82ab8cf..0de388326 100644 --- a/ansible/library/latest_timestamps.py +++ b/ansible/library/latest_timestamps.py @@ -52,21 +52,17 @@ def run_module(): ) timestamps = dict(module.params['repos_dict']) - for index in timestamps: - for repo in timestamps[index]: - for version in timestamps[index][repo]: + for repo in timestamps: + for version in timestamps[repo]: - html_txt = requests.get( - url= module.params['content_url'] + '/' + timestamps[index][repo][version]['pulp_path'] - ).text - timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html - timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags - latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed - timestamps[index][repo][version]['pulp_timestamp'] = latest_timestamp - # timestamps[index] = dict(sorted(timestamps[index])) - - - result['timestamps'] = timestamps + html_txt = requests.get( + url= module.params['content_url'] + '/' + timestamps[repo][version]['pulp_path'] + ).text + timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html + timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags + latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed + timestamps[repo][version]['pulp_timestamp'] = latest_timestamp + result['timestamps'] = dict(sorted(timestamps.items())) module.exit_json(**result) From 1405f3dcedd4416bc568abd9cae3606414d1f203 Mon Sep 17 00:00:00 2001 From: wtripp180901 <78219569+wtripp180901@users.noreply.github.com> Date: Mon, 1 Sep 2025 12:09:09 +0100 Subject: [PATCH 24/30] review suggestions Co-authored-by: Steve Brasier <33413598+sjpb@users.noreply.github.com> --- ansible/roles/dnf_repos/README.md | 5 ++--- environments/site/inventory/groups | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ansible/roles/dnf_repos/README.md b/ansible/roles/dnf_repos/README.md index c98949a87..d02277d88 100644 --- a/ansible/roles/dnf_repos/README.md +++ b/ansible/roles/dnf_repos/README.md @@ -17,7 +17,7 @@ Variables in this role are also required by `pulp_site` so set in - `dnf_repos_all`: Dict of dicts containing information to construct URLs for timestamped repos from Ark for each Rocky version. For example: ``` dnf_repos_all: - appstream: # yum_repository:name + appstream: # ansible.builtin.yum_repository:name '8.10': # ansible_distribution_version or ansible_distribution_major_version repo_file: Rocky-AppStream # yum_repository: file # repo_name: # optional, override yum_repository:name @@ -38,8 +38,7 @@ Variables in this role are also required by `pulp_site` so set in - `dnf_repos_default_epel`: Dict of repos objects following same format as `dnf_repos_all` but only including `epel` repo. - `dnf_repos_pulp_content_url`: Optional str. Content URL of Pulp server to use Ark snapshots from. - Should be overriden if using local Pulp server instead of upstream Ark Pulp (See - `ansible/roles/pulp_site`). Defaults to `{{ appliances_pulp_url }}/pulp/content` + Defaults to `{{ appliances_pulp_url }}/pulp/content` - `dnf_repos_username`: Optional str. Username for Ark. Should be set if using upstream StackHPC Ark Pulp server, but omitted if using local Pulp server (see `ansible/roles/pulp_site`) - `dnf_repos_password`: Optional str. Password for Ark. Should be set if using upstream StackHPC Ark diff --git a/environments/site/inventory/groups b/environments/site/inventory/groups index d30eb10a3..a8ba9ebcc 100644 --- a/environments/site/inventory/groups +++ b/environments/site/inventory/groups @@ -162,4 +162,4 @@ compute # Host to deploy a Pulp server on and sync with mirrors of upstream Ark repositories. Should be a group containing a single VM provisioned # separately from the appliance. e.g # pulp_host ansible_host= -# Note the host name can't conflict with group names i.e can't be called `pulp` or `pulp_server` +# Note inventory host name cannot conflict with group names i.e can't be called `pulp` or `pulp_server`. From dff68b4092ba3bd54eb1f15beb0e864535f7c0f8 Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Mon, 1 Sep 2025 13:01:22 +0100 Subject: [PATCH 25/30] updated README --- ansible/roles/dnf_repos/README.md | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/ansible/roles/dnf_repos/README.md b/ansible/roles/dnf_repos/README.md index d02277d88..2a5838e24 100644 --- a/ansible/roles/dnf_repos/README.md +++ b/ansible/roles/dnf_repos/README.md @@ -1,12 +1,13 @@ dnf_repos ========= -Modifies repo definitions for repofiles in `/etc/yum.repos.d` to point to snapshots in StackHPC's Ark Pulp server. +Modifies repo definitions for repofiles in `/etc/yum.repos.d` to point to snapshots in StackHPC's Ark Pulp server or mirrors of them +on a local Pulp server. Requirements ------------ -Requires Ark credentials. +Requires Ark credentials if using StackHPC's upstream Ark server. Role Variables -------------- @@ -14,29 +15,23 @@ Role Variables Variables in this role are also required by `pulp_site` so set in `environments/common/inventory/groups_vars/all/dnf_repos.yml`. See that file for detailed default values. -- `dnf_repos_all`: Dict of dicts containing information to construct URLs for timestamped repos from Ark for each Rocky version. For example: +- `dnf_repos_repos`: Dict of dicts containing information to construct URLs for Ark snapshots from the target Pulp server for each Rocky version. For example: ``` - dnf_repos_all: + dnf_repos_repos: appstream: # ansible.builtin.yum_repository:name '8.10': # ansible_distribution_version or ansible_distribution_major_version repo_file: Rocky-AppStream # yum_repository: file # repo_name: # optional, override yum_repository:name - pulp_path: rocky/8.10/AppStream/x86_64/os + pulp_path: rocky/8.10/AppStream/x86_64/os # The subpath of the the upstream Ark server's content endpoint URL for the repo's snapshots, see https://ark.stackhpc.com/pulp/content/ pulp_timestamp: 20250614T013846 # pulp_content_url: # optional, dnf_repos_pulp_content_url '9.6': ... ``` -- `dnf_repos_default`: Appliance default repos to use Ark snapshots for. Follows same format as - `dnf_repos_all`, but includes top level keys to allow repos to be conditionally included in - `dnf_repos_all`. See `environments/common/inventory/group_vars/all/dnf_repos.yml` and - `environments/common/inventory/group_vars/all/timestamps.yml` for full templating logic. +- `dnf_repos_default`: Appliance default repos to use Ark snapshots for. Following same format as `dnf_repos_repos`. + See for appliance default repo list `environments/common/inventory/group_vars/all/timestamps.yml`. - `dnf_repos_extra`: Additional repos to use Ark snapshots for. Follows same format as - `dnf_repos_all`. Defaults to `{}` -- `dnf_repos_no_epel`: Dict of all repos included in `dnf_repos_all` excluding - `epel`, used to prevent conflicts with repofile installed by `epel-release` -- `dnf_repos_default_epel`: Dict of repos objects following same format as `dnf_repos_all` but only - including `epel` repo. + `dnf_repos_repos`. Defaults to `{}` - `dnf_repos_pulp_content_url`: Optional str. Content URL of Pulp server to use Ark snapshots from. Defaults to `{{ appliances_pulp_url }}/pulp/content` - `dnf_repos_username`: Optional str. Username for Ark. Should be set if using upstream StackHPC Ark From 7253ad0e8e28b8d1e277717257d832eb9216f6cb Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Mon, 1 Sep 2025 13:30:33 +0100 Subject: [PATCH 26/30] docs tweaks --- ansible/adhoc/deploy-pulp.yml | 2 -- ansible/roles/pulp_site/README.md | 3 ++- .../pulp_site/filter_plugins/pulp-list-filters.py | 2 +- docs/experimental/pulp.md | 15 +++++++++------ .../common/inventory/group_vars/all/dnf_repos.yml | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ansible/adhoc/deploy-pulp.yml b/ansible/adhoc/deploy-pulp.yml index f68259698..f7bafc3e0 100644 --- a/ansible/adhoc/deploy-pulp.yml +++ b/ansible/adhoc/deploy-pulp.yml @@ -1,5 +1,3 @@ -# Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=" - - name: Install pulp on server become: yes hosts: pulp_server diff --git a/ansible/roles/pulp_site/README.md b/ansible/roles/pulp_site/README.md index 086053fab..3af801cd2 100644 --- a/ansible/roles/pulp_site/README.md +++ b/ansible/roles/pulp_site/README.md @@ -8,7 +8,8 @@ Requirements ------------ Requires Ark credentials. The VM you are deploying Pulp on must allow ingress on `pulp_site_port` -and not be externally accessible (as the Pulp server's content is unauthenticated). +and not be externally accessible (as the Pulp server's content is unauthenticated). Rocky Linux 9 has been +tested as the target VM for deploying Pulp. Role Variables -------------- diff --git a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py index 84cb80b34..41e995c82 100644 --- a/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py +++ b/ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py @@ -7,7 +7,7 @@ def filters(self): 'select_repos': self.select_repos, } - def select_repos(self, dnf_repos, target_distro_ver): #TODO: why does baseos get a major and minor version? + def select_repos(self, dnf_repos, target_distro_ver): """ Filter dnf_repos to only those for a relevant distribution version (M.m or M). Returns a list of dicts. Also adds pulp_repo_name field to give the repository a unique name in Pulp to be referenced by subsequent filters diff --git a/docs/experimental/pulp.md b/docs/experimental/pulp.md index 702003dcf..b8a45d46e 100644 --- a/docs/experimental/pulp.md +++ b/docs/experimental/pulp.md @@ -5,21 +5,24 @@ In order to ensure reproducible builds, the appliance can build images using rep ## Deploying/configuring Pulp Server ### Deploying a Pulp server -A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. To use this, create an inventory file defining a group `pulp_server` containing a single host, which requires at least 2 vCPUs and 4GB RAM. Deploying and syncing Pulp has been tested on an RL9 host. The hostvar `ansible_host` should be defined, giving the IP address Ansible should use for ssh. For example: +A playbook is provided to install and configure a Pulp server on a given host. Admin credentials for this server are automatically generated through the `ansible/adhoc/generate-passwords.yml` playbook. To use this, create an inventory file +defining a group `pulp_server` containing a single host, which requires at least 2 vCPUs and 4GB RAM. The group should be defined in your `site` environment's inventory so that a single Pulp server is shared between all environments and +the same snapshots are tested in staging and production. +Deploying and syncing Pulp has been tested on an RL9 host. The hostvar `ansible_host` should be defined, giving the IP address Ansible should use for ssh. For example, you can create an ini file at `environments/site/inventory/pulp` with the contents: ``` [pulp_server] -pulp_host ansible_host= # Note the host name can't conflict with group names i.e can't be called `pulp` or `pulp_server` +pulp_host ansible_host= ``` -Once complete, it will print a message giving a value to set for `appliances_pulp_url`, assuming the `ansible_host` address is also the address the cluster +> [!WARNING] +> The inventory hostname cannot conflict with group names i.e can't be called `pulp` or `pulp_server`. + +Once complete, it will print a message giving a value to set for `appliances_pulp_url` (see example config below), assuming the `ansible_host` address is also the address the cluster should use to reach the Pulp server. Note access to this server's content isn't authenticated so this assumes the `pulp_server` host is not externally reachable. -**TODO: You can actually do this using additional_nodes now, how would we make the pulp store persistant?** -**TODO: don't advise that, we want single server for all environments** - ### Using an existing Pulp server An existing Pulp server can be used to host Ark repos by overriding `pulp_site_password` and `appliances_pulp_url` in the target environment. Note that this assumes the same configuration as the appliance deployed Pulp i.e no content authentication. diff --git a/environments/common/inventory/group_vars/all/dnf_repos.yml b/environments/common/inventory/group_vars/all/dnf_repos.yml index d920ed0de..8ac3887f5 100644 --- a/environments/common/inventory/group_vars/all/dnf_repos.yml +++ b/environments/common/inventory/group_vars/all/dnf_repos.yml @@ -1,4 +1,4 @@ -# dnf_repos_default: see timestamps.yml +# dnf_repos_default: see role ansible/roles/dnf_repos/README.md for format and timestamps.yml for default definition # override this in environments/site/inventory/group_vars/dnf_repos.yml to add repos: dnf_repos_extra: {} From f5f2c0bf638a1453c74339eed3d3e879541d476c Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Mon, 1 Sep 2025 14:01:05 +0100 Subject: [PATCH 27/30] regularised group names --- ansible/fatimage.yml | 2 +- docs/experimental/pulp.md | 1 + environments/common/inventory/groups | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ansible/fatimage.yml b/ansible/fatimage.yml index 839c8dc4a..46a99bc03 100644 --- a/ansible/fatimage.yml +++ b/ansible/fatimage.yml @@ -18,7 +18,7 @@ when: hook_path | exists - name: Sync pulp repos with upstream - hosts: pulp + hosts: pulp_site tasks: - ansible.builtin.include_role: name: pulp_site diff --git a/docs/experimental/pulp.md b/docs/experimental/pulp.md index b8a45d46e..1c24b77d9 100644 --- a/docs/experimental/pulp.md +++ b/docs/experimental/pulp.md @@ -35,6 +35,7 @@ Content can also be synced by running `ansible/adhoc/sync-pulp.yml`. By default ## Example config in site variables ``` +# environments/site/inventory/group_vars/all/pulp_site.yml: appliances_pulp_url: "http://:8080" pulp_site_upstream_username: pulp_site_upstream_password: diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups index bf734f7f4..419fa55a4 100644 --- a/environments/common/inventory/groups +++ b/environments/common/inventory/groups @@ -197,7 +197,7 @@ k3s_agent builder extra_packages -[pulp] +[pulp_site] # Add builder to this group to enable automatically syncing of pulp during image build [cacerts] From ab2cfdff008f7fc54eb469910f4f9e855ccca9b9 Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Mon, 1 Sep 2025 15:43:50 +0100 Subject: [PATCH 28/30] updated operations guide for functionality requiring additional installs --- docs/environments.md | 5 ++++- docs/experimental/pulp.md | 13 +++++++++++-- docs/operations.md | 20 +++++++++++++++++++- environments/site/inventory/groups | 3 +++ 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/docs/environments.md b/docs/environments.md index d1c492312..183b775ee 100644 --- a/docs/environments.md +++ b/docs/environments.md @@ -14,7 +14,10 @@ All environments load the inventory from the `common` environment first, with th The ansible inventory for the environment is in `environments//inventory/`. It should generally contain: - A `hosts` file. This defines the hosts in the appliance. Generally it should be templated out by the deployment automation so it is also a convenient place to define variables which depend on the deployed hosts such as connection variables, IP addresses, ssh proxy arguments etc. -- A `groups` file defining ansible groups, which essentially controls which features of the appliance are enabled and where they are deployed. This repository generally follows a convention where functionality is defined using ansible roles applied to a group of the same name, e.g. `openhpc` or `grafana`. The meaning and use of each group is described in comments in `environments/common/inventory/groups`. As the groups defined there for the common environment are empty, functionality is disabled by default and must be enabled in a specific environment's `groups` file. Two template examples are provided in `environments/commmon/layouts/` demonstrating a minimal appliance with only the Slurm cluster itself, and an appliance with all functionality. +- A `groups` file defining ansible groups, which essentially controls which features of the appliance are enabled and where they are deployed. This repository generally follows a convention where functionality is defined using ansible roles applied to a group +of the same name, e.g. `openhpc` or `grafana`. The meaning and use of each group is described in comments in `environments/common/inventory/groups`. As the groups defined there for the common environment are empty, functionality is disabled by default and must be +enabled in a specific environment's `groups` file. The `site` environment contains an ini file at `environments/site/inventory/groups` which enables groups for default appliance functionality across all environments. Additional groups should generally also be +enabled in this file to avoid divergence between staging and production environments. Note that enabling some groups may require a site-specific image build and Ark credentials (see [operations guide](operations.md)). - Optionally, group variable files in `group_vars//overrides.yml`, where the group names match the functional groups described above. These can be used to override the default configuration for each functionality, which are defined in `environments/common/inventory/group_vars/all/.yml` (the use of `all` here is due to ansible's precedence rules). Although most of the inventory uses the group convention described above there are a few special cases: diff --git a/docs/experimental/pulp.md b/docs/experimental/pulp.md index 1c24b77d9..582eec9d7 100644 --- a/docs/experimental/pulp.md +++ b/docs/experimental/pulp.md @@ -16,7 +16,7 @@ pulp_host ansible_host= ``` > [!WARNING] -> The inventory hostname cannot conflict with group names i.e can't be called `pulp` or `pulp_server`. +> The inventory hostname cannot conflict with group names i.e can't be called `pulp_site` or `pulp_server`. Once complete, it will print a message giving a value to set for `appliances_pulp_url` (see example config below), assuming the `ansible_host` address is also the address the cluster should use to reach the Pulp server. @@ -28,7 +28,7 @@ An existing Pulp server can be used to host Ark repos by overriding `pulp_site_p ## Syncing Pulp content with Ark -If the `pulp` group is added to the Packer build groups, the local Pulp server will be synced with Ark on build. You must authenticate with Ark by overriding `pulp_site_upstream_username` and `pulp_site_upstream_password` with your vault encrypted Ark dev credentials. `dnf_repos_username` and `dnf_repos_password` must remain unset to access content from the local Pulp. +If the `pulp_site` group is added to the Packer build groups, the local Pulp server will be synced with Ark on build. You must authenticate with Ark by overriding `pulp_site_upstream_username` and `pulp_site_upstream_password` with your vault encrypted Ark dev credentials. `dnf_repos_username` and `dnf_repos_password` must remain unset to access content from the local Pulp. Content can also be synced by running `ansible/adhoc/sync-pulp.yml`. By default this syncs repositories for the latest version of Rocky supported by the appliance but this can be overridden by setting extra variables for `pulp_site_target_arch`, `pulp_site_target_distribution` and `pulp_site_target_distribution_version`. @@ -40,3 +40,12 @@ appliances_pulp_url: "http://:8080" pulp_site_upstream_username: pulp_site_upstream_password: ``` + +## Installing packages from Pulp at runtime +By default, system repos are overwritten to point at Pulp repos during [image builds,](../image-build.md) so using a site Pulp server will require a new fatimage. If you instead wish to install packages at runtime, +you will need to add all host groups on which you will be installing packages to the `dnf_repos` group in `environments/site/inventory/groups` e.g: + +``` +[dnf_repos:children] +cluster +``` diff --git a/docs/operations.md b/docs/operations.md index 4f7bc5cfb..4c5c640c5 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -9,7 +9,7 @@ All subsequent sections assume that: - Appropriate OpenStack credentials are available. - Any non-appliance controlled infrastructure is available (e.g. networks, volumes, etc.). - `$ENV` is your current, activated environment, as defined by e.g. `environments/production/`. -- `$SITE_ENV` is the base site-specific environment, as defined by e.g. `environments/mysite/`. +- `$SITE_ENV` is the base site-specific environment, as defined by `environments/site/`. - A string `some/path/to/file.yml:myvar` defines a path relative to the repository root and an Ansible variable in that file. - Configuration is generally common to all environments at a site, i.e. is made in `environments/$SITE_ENV` not `environments/$ENV`. @@ -62,6 +62,24 @@ This is a usually a two-step process: Deploying the additional nodes and applying these changes requires rerunning both OpenTofu and the Ansible site.yml playbook - follow [Deploying a Cluster](#Deploying-a-Cluster). +# Enabling additional functionality +Roles in the appliance which are disabled by default can be enabled by adding the appropriate groups as children of the role's corresponding group in `environments/site/inventory/groups`. For example, +to install a Squid proxy on nodes in the login group, you would modify the `squid` group definition in `environments/site/inventory/groups` to: + +``` +[squid:children] +# Hosts to run squid proxy +login +``` + +Note that many non-default roles include package installations from repositories which the appliance overwrites to point at snapshotted mirrors on a Pulp server (by default StackHPC's Ark server), which are +disabled during runtime to prevent Ark credentials from being leaked. To enable this functionality, you must therefore either: + +- Create a site-specific fatimage (see [image build docs](image-build.md)) with the appropriate group added to the `inventory_groups` Packer variables. +- If you instead wish roles to perform their installations during runtime, deploy a site Pulp server and sync it with with mirrors of the snapshots from the upstream Ark server (see [Pulp docs](experimental/pulp.md)). + +In both cases, Ark credentials will be required. + # Adding Additional Packages By default, the following utility packages are installed during the StackHPC image build: - htop diff --git a/environments/site/inventory/groups b/environments/site/inventory/groups index a8ba9ebcc..c6134ff80 100644 --- a/environments/site/inventory/groups +++ b/environments/site/inventory/groups @@ -158,6 +158,9 @@ compute # Note that this feature currently assumes all compute nodes are VMs, enabling # when the cluster contains baremetal compute nodes may lead to unexpected scheduling behaviour +[pulp_site] +# Add builder to this group to enable automatically syncing of pulp during image build + [pulp_server] # Host to deploy a Pulp server on and sync with mirrors of upstream Ark repositories. Should be a group containing a single VM provisioned # separately from the appliance. e.g From edb4e12b8423d67886b1744f8d6fb0d644345e73 Mon Sep 17 00:00:00 2001 From: wtripp180901 <78219569+wtripp180901@users.noreply.github.com> Date: Thu, 4 Sep 2025 12:48:29 +0100 Subject: [PATCH 29/30] review changes from docs Co-authored-by: Steve Brasier <33413598+sjpb@users.noreply.github.com> --- environments/common/inventory/groups | 2 +- environments/site/inventory/groups | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups index 419fa55a4..6926355ed 100644 --- a/environments/common/inventory/groups +++ b/environments/common/inventory/groups @@ -198,7 +198,7 @@ builder extra_packages [pulp_site] -# Add builder to this group to enable automatically syncing of pulp during image build +# Add builder to this group to automatically sync pulp during image build [cacerts] # Hosts to configure CA certificates and trusts on diff --git a/environments/site/inventory/groups b/environments/site/inventory/groups index c6134ff80..b78197df0 100644 --- a/environments/site/inventory/groups +++ b/environments/site/inventory/groups @@ -159,7 +159,7 @@ compute # when the cluster contains baremetal compute nodes may lead to unexpected scheduling behaviour [pulp_site] -# Add builder to this group to enable automatically syncing of pulp during image build +# Add builder to this group to automatically sync pulp during image build [pulp_server] # Host to deploy a Pulp server on and sync with mirrors of upstream Ark repositories. Should be a group containing a single VM provisioned From 412baede4a2288ad81773b8b1d5186a2d089c586 Mon Sep 17 00:00:00 2001 From: wtripp180901 Date: Thu, 4 Sep 2025 12:57:49 +0100 Subject: [PATCH 30/30] renamed timestamps.yml to dnf_repos_timestamps.yml --- ansible/ci/update_timestamps.yml | 2 +- ansible/roles/dnf_repos/README.md | 2 +- .../group_vars/all/{timestamps.yml => dnf_repo_timestamps.yml} | 0 environments/common/inventory/group_vars/all/dnf_repos.yml | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename environments/common/inventory/group_vars/all/{timestamps.yml => dnf_repo_timestamps.yml} (100%) diff --git a/ansible/ci/update_timestamps.yml b/ansible/ci/update_timestamps.yml index b9ab9e0ec..8db475706 100644 --- a/ansible/ci/update_timestamps.yml +++ b/ansible/ci/update_timestamps.yml @@ -8,7 +8,7 @@ - name: Overwrite repo timestamps with latest ansible.builtin.copy: - dest: "{{ appliances_repository_root }}/environments/common/inventory/group_vars/all/timestamps.yml" + dest: "{{ appliances_repository_root }}/environments/common/inventory/group_vars/all/dnf_repo_timestamps.yml" content: "{{ repo_template | to_nice_yaml(indent=2) }}" backup: true vars: diff --git a/ansible/roles/dnf_repos/README.md b/ansible/roles/dnf_repos/README.md index 2a5838e24..ff22c79fc 100644 --- a/ansible/roles/dnf_repos/README.md +++ b/ansible/roles/dnf_repos/README.md @@ -29,7 +29,7 @@ Variables in this role are also required by `pulp_site` so set in ... ``` - `dnf_repos_default`: Appliance default repos to use Ark snapshots for. Following same format as `dnf_repos_repos`. - See for appliance default repo list `environments/common/inventory/group_vars/all/timestamps.yml`. + See for appliance default repo list `environments/common/inventory/group_vars/all/dnf_repo_timestamps.yml`. - `dnf_repos_extra`: Additional repos to use Ark snapshots for. Follows same format as `dnf_repos_repos`. Defaults to `{}` - `dnf_repos_pulp_content_url`: Optional str. Content URL of Pulp server to use Ark snapshots from. diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/dnf_repo_timestamps.yml similarity index 100% rename from environments/common/inventory/group_vars/all/timestamps.yml rename to environments/common/inventory/group_vars/all/dnf_repo_timestamps.yml diff --git a/environments/common/inventory/group_vars/all/dnf_repos.yml b/environments/common/inventory/group_vars/all/dnf_repos.yml index 8ac3887f5..e7a8ace33 100644 --- a/environments/common/inventory/group_vars/all/dnf_repos.yml +++ b/environments/common/inventory/group_vars/all/dnf_repos.yml @@ -1,4 +1,4 @@ -# dnf_repos_default: see role ansible/roles/dnf_repos/README.md for format and timestamps.yml for default definition +# dnf_repos_default: see role ansible/roles/dnf_repos/README.md for format and dnf_repo_timestamps.yml for default definition # override this in environments/site/inventory/group_vars/dnf_repos.yml to add repos: dnf_repos_extra: {}