Refactor Pulp repo definitions and add more Pulp documentation

ansible/adhoc/deploy-pulp.yml

@@ -1,15 +1,8 @@
 # Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<pulp server hostname>"
 
-- name: Add temporary pulp server host
-  hosts: localhost
-  tasks:
-  - ansible.builtin.add_host:
-      name: "{{ pulp_server }}"
-      group: "_pulp_host"
-
-- name: Install pulp on server and add to config
+- name: Install pulp on server
   become: yes
-  hosts: _pulp_host
+  hosts: pulp_server
   tasks:
   - name: Install pulp
     ansible.builtin.include_role:
@@ -22,5 +15,5 @@
     debug:
       msg: | 
         Server configured, override 'appliances_pulp_url' with
-          appliances_pulp_url: "http://{{ pulp_server }}:{{ pulp_site_port }}"
-        in your environments
+          appliances_pulp_url: "http://{{ hostvars[groups['pulp_server'] | first].ansible_host }}:{{ pulp_site_port }}"
+        (or the correct IP if multi-homed) in your environments

ansible/adhoc/sync-pulp.yml

@@ -6,5 +6,4 @@
       vars:
         pulp_site_target_arch: "x86_64"
         pulp_site_target_distribution: "rocky"
-        pulp_site_target_distribution_version: "9.5"
-        pulp_site_target_distribution_version_major: "9"
+        pulp_site_target_distribution_version: "{{ dnf_repos_all['baseos'].keys() | map('float') | sort | last }}"

ansible/ci/update_timestamps.yml

@@ -2,7 +2,7 @@
   tasks:
     - name: Get latest timestamps from sources
       latest_timestamps:
-        repos_dict: "{{ appliances_pulp_repos }}"
+        repos_dict: "{{ dnf_repos_default }}"
         content_url: "https://ark.stackhpc.com/pulp/content"
       register: _result
 
@@ -13,4 +13,4 @@
         backup: true
       vars:
         repo_template:
-          appliances_pulp_repos: "{{ _result.timestamps }}"
+          dnf_repos_default: "{{ _result.timestamps }}"

ansible/filter_plugins/utils.py

@@ -61,11 +61,6 @@ def to_ood_regex(items):
     r = ['(%s)' % v for v in r]
     return '|'.join(r)
 
-def appliances_repo_to_subpath(repo_entry):
-    """ Take an element from appliances_pulp_repos and convert it to a pulp path. This assumes that the remote and local pulp structures are the same
-    """
-    return repo_entry['path'] + '/' + repo_entry['timestamp']
-
 class FilterModule(object):
     ''' Ansible core jinja2 filters '''
 
@@ -81,5 +76,4 @@ def filters(self):
             'exists': exists,
             'warn': self.warn,
             'to_ood_regex': to_ood_regex,
-            'appliances_repo_to_subpath': appliances_repo_to_subpath
         }

ansible/library/latest_timestamps.py

@@ -52,18 +52,21 @@ def run_module():
     )
 
     timestamps = dict(module.params['repos_dict'])
-    for repo in timestamps:
-        for version in timestamps[repo]:
+    for index in timestamps:
+        for repo in timestamps[index]:
+            for version in timestamps[index][repo]:
 
-            html_txt = requests.get(
-                    url= module.params['content_url'] + '/' + timestamps[repo][version]['path']
-                ).text
-            timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html
-            timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags
-            latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
-            timestamps[repo][version]['timestamp'] = latest_timestamp
+                html_txt = requests.get(
+                        url= module.params['content_url'] + '/' + timestamps[index][repo][version]['pulp_path']
+                    ).text
+                timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html
+                timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags
+                latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
+                timestamps[index][repo][version]['pulp_timestamp'] = latest_timestamp
+        # timestamps[index] = dict(sorted(timestamps[index]))
 
-    result['timestamps'] = dict(sorted(timestamps.items()))
+
+    result['timestamps'] = timestamps
 
     module.exit_json(**result)
 

ansible/roles/dnf_repos/README.md

@@ -0,0 +1,46 @@
+dnf_repos
+=========
+
+Modifies repo definitions for repofiles in `/etc/yum.repos.d` to point to snapshots in StackHPC's Ark Pulp server.
+
+Requirements
+------------
+
+Requires Ark credentials.
+
+Role Variables
+--------------
+
+Variables in this role are also required by `pulp_site` so set in 
+`environments/common/inventory/groups_vars/all/dnf_repos.yml`. See that file for detailed default values.
+
+- `dnf_repos_all`: Dict of dicts containing information to construct URLs for timestamped repos from Ark for each Rocky version. For example:
+    ```
+    dnf_repos_all:
+        appstream:                          # yum_repository:name
+            '8.10':                           # ansible_distribution_version or ansible_distribution_major_version
+                repo_file: Rocky-AppStream      # yum_repository: file
+                # repo_name:                    # optional, override yum_repository:name
+                pulp_path: rocky/8.10/AppStream/x86_64/os
+                pulp_timestamp: 20250614T013846
+                # pulp_content_url:             # optional, dnf_repos_pulp_content_url
+            '9.6':
+                ...
+    ```
+- `dnf_repos_default`: Appliance default repos to use Ark snapshots for. Follows same format as
+  `dnf_repos_all`, but includes top level keys to allow repos to be conditionally included in 
+  `dnf_repos_all`. See `environments/common/inventory/group_vars/all/dnf_repos.yml` and
+  `environments/common/inventory/group_vars/all/timestamps.yml` for full templating logic.
+- `dnf_repos_extra`: Additional repos to use Ark snapshots for. Follows same format as
+  `dnf_repos_all`. Defaults to `{}`
+- `dnf_repos_no_epel`: Dict of all repos included in `dnf_repos_all` excluding 
+  `epel`, used to prevent conflicts with repofile installed by `epel-release`
+- `dnf_repos_default_epel`: Dict of repos objects following same format as `dnf_repos_all` but only 
+   including `epel` repo. 
+- `dnf_repos_pulp_content_url`: Optional str. Content URL of Pulp server to use Ark snapshots from. 
+  Should be overriden if using local Pulp server instead of upstream Ark Pulp (See
+  `ansible/roles/pulp_site`). Defaults to `{{ appliances_pulp_url }}/pulp/content`
+- `dnf_repos_username`: Optional str. Username for Ark. Should be set if using upstream StackHPC Ark
+  Pulp server, but omitted if using local Pulp server (see `ansible/roles/pulp_site`)
+- `dnf_repos_password`: Optional str. Password for Ark. Should be set if using upstream StackHPC Ark
+  Pulp server, but omitted if using local Pulp server (see `ansible/roles/pulp_site`)

ansible/roles/dnf_repos/defaults/main.yml

@@ -1,54 +1,8 @@
+dnf_repos_default: {} # see environments/common/inventory/group_vars/all/timestamps.yml
+dnf_repos_extra: {}
+dnf_repos_no_epel: {}
+dnf_repos_all: {}
+dnf_repos_default_epel: {}
 dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content"
 dnf_repos_username: "{{ omit }}"
 dnf_repos_password: "{{ omit }}"
-
-dnf_repos_filenames:
-  '8':
-    baseos: 'Rocky-BaseOS'
-    appstream: 'Rocky-AppStream'
-    crb: 'Rocky-PowerTools'
-    extras: 'Rocky-Extras'
-    grafana: 'grafana'
-  '9':
-    baseos: 'rocky'
-    appstream: 'rocky'
-    crb: 'rocky'
-    extras: 'rocky-extras'
-    grafana: 'grafana'
-
-dnf_repos_version_filenames: "{{ dnf_repos_filenames[ansible_distribution_major_version] }}"
-
-# epel installed separately
-dnf_repos_default_repolist:
-- file: "{{ dnf_repos_version_filenames.baseos }}"
-  name: baseos
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.baseos[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.appstream }}"
-  name: appstream
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.appstream[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.crb }}"
-  name: "{{ 'powertools' if ansible_distribution_major_version == '8' else 'crb' }}"
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.crb[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.extras }}"
-  name: extras
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.extras[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: ceph
-  name: Ceph
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.ceph[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.grafana }}"
-  name: grafana
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.grafana[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-
-dnf_repos_openhpc_repolist:
-- name: OpenHPC
-  file: OpenHPC
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_base[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-- name: OpenHPC-updates
-  file: OpenHPC
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_updates[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-
-dnf_repos_extra_repolist: []
-dnf_repos_repolist: "{{ dnf_repos_default_repolist + (dnf_repos_openhpc_repolist if (openhpc_install_type | default('ohpc')) == 'ohpc' else []) + dnf_repos_extra_repolist }}"
-
-dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.epel[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-dnf_repos_epel_description: "epel"

ansible/roles/dnf_repos/tasks/disable_repos.yml

@@ -1,21 +1,20 @@
 ---
 - name: Remove password and disable Pulp repos
   ansible.builtin.yum_repository:
-    file: "{{ item.file }}"
-    name: "{{ item.name }}"
-    baseurl: "{{ item.base_url }}"
-    description: "{{ item.name }}"
+    file: "{{ repo_values.repo_file }}"
+    name: "{{ repo_name }}"
+    baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}"
+    description: "{{ repo_name }}"
     enabled: false
-  loop: "{{ dnf_repos_repolist }}"
-
-- name: Remove password and disable EPEL repo
-  ansible.builtin.yum_repository:
-    name: epel
-    file: epel
-    description: "{{ dnf_repos_epel_description }}"
-    baseurl: "{{ dnf_repos_epel_baseurl }}"
     gpgcheck: false
-    enabled: false
+  loop: "{{ dnf_repos_all | dict2items }}"
+  loop_control:
+    label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
+  vars:
+    repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}"
+    repo_values: "{{ item.value[repo_os] }}"
+    repo_name: "{{ repo_values.repo_name | default(item.key) }}"
+    repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}"
 
 - name: Get all repo files
   ansible.builtin.find:

ansible/roles/dnf_repos/tasks/set_repos.yml

@@ -2,26 +2,41 @@
 
 - name: Replace system repos with Pulp repos
   ansible.builtin.yum_repository:
-    file: "{{ item.file }}"
-    name: "{{ item.name }}"
-    baseurl: "{{ item.base_url }}"
-    description: "{{ item.name }}"
+    file: "{{ repo_values.repo_file }}"
+    name: "{{ repo_name }}"
+    baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}"
+    description: "{{ repo_name }}"
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
     gpgcheck: false
-  loop: "{{ dnf_repos_repolist }}"
+  loop: "{{ dnf_repos_no_epel | dict2items }}"
+  loop_control:
+    label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
+  vars:
+    repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}"
+    repo_values: "{{ item.value[repo_os] }}"
+    repo_name: "{{ repo_values.repo_name | default(item.key) }}"
+    repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}"
 
 - name: Install epel-release
-  # done so that roles installing epel via epel-release don't over-write our changes to the epel repo
+  # So roles installing epel via epel-release don't overwrite changes to the epel repo below
   ansible.builtin.dnf:
     name: epel-release
 
-- name: Use Pulp EPEL repo
+- name: Replace epel repo with Pulp repo
   ansible.builtin.yum_repository:
-    name: epel
-    file: epel
-    description: "{{ dnf_repos_epel_description }}"
-    gpgcheck: false
-    baseurl: "{{ dnf_repos_epel_baseurl }}"
+    file: "{{ repo_values.repo_file }}"
+    name: "{{ repo_name }}"
+    baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}"
+    description: "{{ repo_name }}"
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
+    gpgcheck: false
+  loop: "{{ dnf_repos_default_epel | dict2items }}"
+  loop_control:
+    label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
+  vars:
+    repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}"
+    repo_values: "{{ item.value[repo_os] }}"
+    repo_name: "{{ repo_values.repo_name | default(item.key) }}"
+    repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}"

ansible/roles/pulp_site/README.md

@@ -0,0 +1,35 @@
+pulp_site
+=========
+
+Contains playbooks to deploy a Pulp server and sync its content with repo snapshots in
+StackHPC's Ark Pulp server
+
+Requirements
+------------
+
+Requires Ark credentials. The VM you are deploying Pulp on must allow ingress on `pulp_site_port`
+and not be externally accessible (as the Pulp server's content is unauthenticated).
+
+Role Variables
+--------------
+
+- `pulp_site_url`: Required str. The base url from which Pulp content will be hosted. Defaults to `{{ appliances_pulp_url }}`. 
+                 Value to set for ``appliances_pulp_url` will be generated and output by the deploy.yml playbook.
+- `pulp_site_port`: Optional str. Port to serve Pulp server on. Defaults to `8080`.
+- `pulp_site_username`: Optional str. Admin username for the Pulp server. Defaults to `admin`.
+- `pulp_site_password`: Required str. Admin password for the Pulp server. Defaults to `{{ vault_pulp_admin_password }}`.
+- `pulp_site_upstream_username`: Required str. Username for accessing content from the upstream Ark Pulp server.
+- `pulp_site_upstream_password`: Required str. Password for upstream Ark Pulp server.
+- `pulp_site_upstream_content_url`: Optional str. Content URL of upstream Ark Pulp. Defaults to `https://ark.stackhpc.com/pulp/content`.
+- `pulp_site_install_dir`: Optional str. Directory on Pulp host to install config and persistent state to be mounted into Pulp container. Defaults to `/home/rocky/pulp`.
+- `pulp_site_target_facts`: Optional str. The `ansible_facts` of a host which will be pulling from your Pulp server, allowing the role to auto-discover the necessary repos to pull.
+                          defaults to `{{ hostvars[groups['pulp'][0]]['ansible_facts'] }}`.
+- `pulp_site_target_distribution_version`: Optional str. The Rocky Linux minor release to sync repos from Ark for. Defaults to `{{ pulp_site_target_facts['distribution_version'] }}`.
+- `pulp_site_rpm_repo_defaults`: Optional dict. Contains key value pairs for fields which are common to all repo definition in `pulp_site_rpm_repos`. Includes values for `remote_username`,
+                               `remote_password` and `policy` by default.
+- `pulp_site_rpm_repos`: Optional list of dicts. List of repo definitions in format required by the `stackhpc.pulp.pulp_repository`. Defaults to modified versions of repos defined in
+                       `dnf_repos_all`.
+- `pulp_site_rpm_publications`: Optional list of dicts. List of repo definitions in format required by the `stackhpc.pulp.pulp_publication`. Defaults to list of publications for repos defined in
+                              `dnf_repos_all`.
+- `pulp_site_rpm_distributions`: Optional list of dicts. List of repo definitions in format required by the `stackhpc.pulp.pulp_distribution`. Defaults to list of distributions for repos defined in
+                              `dnf_repos_all`.