Refactor Pulp repo definitions and add more Pulp documentation (#760)

* WIP: refactor repos definitions

* add more repos and cope with CRB/PowerTools oddness

* add epel

* use pulp_server as a group

* add epel default

* wip: get pulp sync working

* fixed sync

* autodetect latest in adhoc script, refactored timestamps to allow gated ohpc repos, fixed pulp site

* fixed distributions + ohpc repos

* updated timestamps script + bumped rocky 9 timestamps

* removed pulp_repo_name fields

* updated docs, added gpg checks, simplified filters

* Added pulp systemd file + removed unused vars

* added READMEs + updated variable names

* disabled gpg checks for dnf_repos

* typo

* fixed disable repos task

* bump images

* remove dnf_repos extra index/key and make epel/openhpc special-cases simpler

* clarify pulp distro selection

* fixup sync vars

* fixup grafana vars

* revert latest timestamp changes for extra key level

* review suggestions

Co-authored-by: Steve Brasier <[email protected]>

* updated README

* docs tweaks

* regularised group names

* updated operations guide for functionality requiring additional installs

* review changes from docs

Co-authored-by: Steve Brasier <[email protected]>

* renamed timestamps.yml to dnf_repos_timestamps.yml

---------

Co-authored-by: Steve Brasier <[email protected]>
Co-authored-by: Steve Brasier <[email protected]>

Author: wtripp180901

ansible/adhoc/deploy-pulp.yml

@@ -1,15 +1,6 @@
-# Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<pulp server hostname>"
-
-- name: Add temporary pulp server host
-  hosts: localhost
-  tasks:
-  - ansible.builtin.add_host:
-      name: "{{ pulp_server }}"
-      group: "_pulp_host"
-
-- name: Install pulp on server and add to config
+- name: Install pulp on server
   become: yes
-  hosts: _pulp_host
+  hosts: pulp_server
   tasks:
   - name: Install pulp
     ansible.builtin.include_role:
@@ -22,5 +13,5 @@
     debug:
       msg: | 
         Server configured, override 'appliances_pulp_url' with
-          appliances_pulp_url: "http://{{ pulp_server }}:{{ pulp_site_port }}"
-        in your environments
+          appliances_pulp_url: "http://{{ hostvars[groups['pulp_server'] | first].ansible_host }}:{{ pulp_site_port }}"
+        (or the correct IP if multi-homed) in your environments

ansible/adhoc/sync-pulp.yml

@@ -6,5 +6,5 @@
       vars:
         pulp_site_target_arch: "x86_64"
         pulp_site_target_distribution: "rocky"
-        pulp_site_target_distribution_version: "9.5"
-        pulp_site_target_distribution_version_major: "9"
+        # default distribution to *latest* specified for baseos repo:
+        pulp_site_target_distribution_version: "{{ dnf_repos_repos['baseos'].keys() | map('float') | sort | last }}"

ansible/ci/update_timestamps.yml

@@ -2,15 +2,15 @@
   tasks:
     - name: Get latest timestamps from sources
       latest_timestamps:
-        repos_dict: "{{ appliances_pulp_repos }}"
+        repos_dict: "{{ dnf_repos_default }}"
         content_url: "https://ark.stackhpc.com/pulp/content"
       register: _result
 
     - name: Overwrite repo timestamps with latest
       ansible.builtin.copy:
-        dest: "{{ appliances_repository_root }}/environments/common/inventory/group_vars/all/timestamps.yml"
+        dest: "{{ appliances_repository_root }}/environments/common/inventory/group_vars/all/dnf_repo_timestamps.yml"
         content: "{{ repo_template | to_nice_yaml(indent=2) }}"
         backup: true
       vars:
         repo_template:
-          appliances_pulp_repos: "{{ _result.timestamps }}"
+          dnf_repos_default: "{{ _result.timestamps }}"

ansible/fatimage.yml

@@ -18,7 +18,7 @@
   when: hook_path | exists
 
 - name: Sync pulp repos with upstream
-  hosts: pulp
+  hosts: pulp_site
   tasks:
   - ansible.builtin.include_role:
       name: pulp_site

ansible/filter_plugins/utils.py

@@ -61,11 +61,6 @@ def to_ood_regex(items):
     r = ['(%s)' % v for v in r]
     return '|'.join(r)
 
-def appliances_repo_to_subpath(repo_entry):
-    """ Take an element from appliances_pulp_repos and convert it to a pulp path. This assumes that the remote and local pulp structures are the same
-    """
-    return repo_entry['path'] + '/' + repo_entry['timestamp']
-
 class FilterModule(object):
     ''' Ansible core jinja2 filters '''
 
@@ -81,5 +76,4 @@ def filters(self):
             'exists': exists,
             'warn': self.warn,
             'to_ood_regex': to_ood_regex,
-            'appliances_repo_to_subpath': appliances_repo_to_subpath
         }

ansible/library/latest_timestamps.py

@@ -56,13 +56,12 @@ def run_module():
         for version in timestamps[repo]:
 
             html_txt = requests.get(
-                    url= module.params['content_url'] + '/' + timestamps[repo][version]['path']
+                    url= module.params['content_url'] + '/' + timestamps[repo][version]['pulp_path']
                 ).text
             timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html
             timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags
             latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
-            timestamps[repo][version]['timestamp'] = latest_timestamp
-            
+            timestamps[repo][version]['pulp_timestamp'] = latest_timestamp
     result['timestamps'] = dict(sorted(timestamps.items()))
 
     module.exit_json(**result)

ansible/roles/dnf_repos/README.md

@@ -0,0 +1,40 @@
+dnf_repos
+=========
+
+Modifies repo definitions for repofiles in `/etc/yum.repos.d` to point to snapshots in StackHPC's Ark Pulp server or mirrors of them
+on a local Pulp server.
+
+Requirements
+------------
+
+Requires Ark credentials if using StackHPC's upstream Ark server.
+
+Role Variables
+--------------
+
+Variables in this role are also required by `pulp_site` so set in 
+`environments/common/inventory/groups_vars/all/dnf_repos.yml`. See that file for detailed default values.
+
+- `dnf_repos_repos`: Dict of dicts containing information to construct URLs for Ark snapshots from the target Pulp server for each Rocky version. For example:
+    ```
+    dnf_repos_repos:
+        appstream:                          # ansible.builtin.yum_repository:name
+            '8.10':                           # ansible_distribution_version or ansible_distribution_major_version
+                repo_file: Rocky-AppStream      # yum_repository: file
+                # repo_name:                    # optional, override yum_repository:name
+                pulp_path: rocky/8.10/AppStream/x86_64/os # The subpath of the the upstream Ark server's content endpoint URL for the repo's snapshots, see https://ark.stackhpc.com/pulp/content/
+                pulp_timestamp: 20250614T013846
+                # pulp_content_url:             # optional, dnf_repos_pulp_content_url
+            '9.6':
+                ...
+    ```
+- `dnf_repos_default`: Appliance default repos to use Ark snapshots for. Following same format as `dnf_repos_repos`.
+  See for appliance default repo list `environments/common/inventory/group_vars/all/dnf_repo_timestamps.yml`.
+- `dnf_repos_extra`: Additional repos to use Ark snapshots for. Follows same format as
+  `dnf_repos_repos`. Defaults to `{}`
+- `dnf_repos_pulp_content_url`: Optional str. Content URL of Pulp server to use Ark snapshots from. 
+  Defaults to `{{ appliances_pulp_url }}/pulp/content`
+- `dnf_repos_username`: Optional str. Username for Ark. Should be set if using upstream StackHPC Ark
+  Pulp server, but omitted if using local Pulp server (see `ansible/roles/pulp_site`)
+- `dnf_repos_password`: Optional str. Password for Ark. Should be set if using upstream StackHPC Ark
+  Pulp server, but omitted if using local Pulp server (see `ansible/roles/pulp_site`)

ansible/roles/dnf_repos/defaults/main.yml

@@ -1,54 +1,4 @@
+dnf_repos_repos: {} # see environments/common/inventory/group_vars/all/{dnf_repos,timestamps}.yml
 dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content"
 dnf_repos_username: "{{ omit }}"
 dnf_repos_password: "{{ omit }}"
-
-dnf_repos_filenames:
-  '8':
-    baseos: 'Rocky-BaseOS'
-    appstream: 'Rocky-AppStream'
-    crb: 'Rocky-PowerTools'
-    extras: 'Rocky-Extras'
-    grafana: 'grafana'
-  '9':
-    baseos: 'rocky'
-    appstream: 'rocky'
-    crb: 'rocky'
-    extras: 'rocky-extras'
-    grafana: 'grafana'
-
-dnf_repos_version_filenames: "{{ dnf_repos_filenames[ansible_distribution_major_version] }}"
-
-# epel installed separately
-dnf_repos_default_repolist:
-- file: "{{ dnf_repos_version_filenames.baseos }}"
-  name: baseos
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.baseos[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.appstream }}"
-  name: appstream
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.appstream[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.crb }}"
-  name: "{{ 'powertools' if ansible_distribution_major_version == '8' else 'crb' }}"
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.crb[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.extras }}"
-  name: extras
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.extras[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: ceph
-  name: Ceph
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.ceph[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.grafana }}"
-  name: grafana
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.grafana[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-
-dnf_repos_openhpc_repolist:
-- name: OpenHPC
-  file: OpenHPC
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_base[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-- name: OpenHPC-updates
-  file: OpenHPC
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_updates[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-
-dnf_repos_extra_repolist: []
-dnf_repos_repolist: "{{ dnf_repos_default_repolist + (dnf_repos_openhpc_repolist if (openhpc_install_type | default('ohpc')) == 'ohpc' else []) + dnf_repos_extra_repolist }}"
-
-dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.epel[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-dnf_repos_epel_description: "epel"

ansible/roles/dnf_repos/tasks/disable_repos.yml

@@ -1,21 +1,20 @@
 ---
 - name: Remove password and disable Pulp repos
   ansible.builtin.yum_repository:
-    file: "{{ item.file }}"
-    name: "{{ item.name }}"
-    baseurl: "{{ item.base_url }}"
-    description: "{{ item.name }}"
+    file: "{{ repo_values.repo_file }}"
+    name: "{{ repo_name }}"
+    baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}"
+    description: "{{ repo_name }}"
     enabled: false
-  loop: "{{ dnf_repos_repolist }}"
-
-- name: Remove password and disable EPEL repo
-  ansible.builtin.yum_repository:
-    name: epel
-    file: epel
-    description: "{{ dnf_repos_epel_description }}"
-    baseurl: "{{ dnf_repos_epel_baseurl }}"
     gpgcheck: false
-    enabled: false
+  loop: "{{ dnf_repos_repos | dict2items }}"
+  loop_control:
+    label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
+  vars:
+    repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}"
+    repo_values: "{{ item.value[repo_os] }}"
+    repo_name: "{{ repo_values.repo_name | default(item.key) }}"
+    repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}"
 
 - name: Get all repo files
   ansible.builtin.find:

ansible/roles/dnf_repos/tasks/set_repos.yml

@@ -1,27 +1,44 @@
 ---
 
-- name: Replace system repos with Pulp repos
+- name: Replace non-epel repos with Pulp repos
   ansible.builtin.yum_repository:
-    file: "{{ item.file }}"
-    name: "{{ item.name }}"
-    baseurl: "{{ item.base_url }}"
-    description: "{{ item.name }}"
+    file: "{{ repo_values.repo_file }}"
+    name: "{{ repo_name }}"
+    baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}"
+    description: "{{ repo_name }}"
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
     gpgcheck: false
-  loop: "{{ dnf_repos_repolist }}"
+  loop: "{{ dnf_repos_repos | dict2items }}"
+  loop_control:
+    label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
+  when: repo_name != 'epel'
+  vars:
+    repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}"
+    repo_values: "{{ item.value[repo_os] }}"
+    repo_name: "{{ repo_values.repo_name | default(item.key) }}"
+    repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}"
 
 - name: Install epel-release
-  # done so that roles installing epel via epel-release don't over-write our changes to the epel repo
+  # So roles installing epel via epel-release don't overwrite changes to the epel repo below
   ansible.builtin.dnf:
     name: epel-release
 
-- name: Use Pulp EPEL repo
+- name: Replace epel repo with Pulp repo
   ansible.builtin.yum_repository:
-    name: epel
-    file: epel
-    description: "{{ dnf_repos_epel_description }}"
-    gpgcheck: false
-    baseurl: "{{ dnf_repos_epel_baseurl }}"
+    file: "{{ repo_values.repo_file }}"
+    name: "{{ repo_name }}"
+    baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}"
+    description: "{{ repo_name }}"
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
+    gpgcheck: false
+  loop: "{{ dnf_repos_repos | dict2items }}"
+  loop_control:
+    label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
+  when: repo_name == 'epel'
+  vars:
+    repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}"
+    repo_values: "{{ item.value[repo_os] }}"
+    repo_name: "{{ repo_values.repo_name | default(item.key) }}"
+    repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}"