Refactor Pulp repo definitions and add more Pulp documentation

ansible/adhoc/deploy-pulp.yml

@@ -1,15 +1,8 @@
 # Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<pulp server hostname>"
 
-- name: Add temporary pulp server host
-  hosts: localhost
-  tasks:
-  - ansible.builtin.add_host:
-      name: "{{ pulp_server }}"
-      group: "_pulp_host"
-
-- name: Install pulp on server and add to config
+- name: Install pulp on server
   become: yes
-  hosts: _pulp_host
+  hosts: pulp_server # TODO: add this to common/groups
   tasks:
   - name: Install pulp
     ansible.builtin.include_role:
@@ -22,5 +15,5 @@
     debug:
       msg: | 
         Server configured, override 'appliances_pulp_url' with
-          appliances_pulp_url: "http://{{ pulp_server }}:{{ pulp_site_port }}"
-        in your environments
+          appliances_pulp_url: "http://{{ hostvars[groups['pulp_server'] | first].ansible_host }}:{{ pulp_site_port }}"
+        (or the correct IP if multi-homed) in your environments

ansible/adhoc/sync-pulp.yml

@@ -6,5 +6,4 @@
       vars:
         pulp_site_target_arch: "x86_64"
         pulp_site_target_distribution: "rocky"
-        pulp_site_target_distribution_version: "9.5"
-        pulp_site_target_distribution_version_major: "9"
+        pulp_site_target_distribution_version: "{{ dnf_repos_all['baseos'].keys() | map('float') | sort | last }}"

ansible/ci/update_timestamps.yml

@@ -2,7 +2,7 @@
   tasks:
     - name: Get latest timestamps from sources
       latest_timestamps:
-        repos_dict: "{{ appliances_pulp_repos }}"
+        repos_dict: "{{ dnf_repos_default }}"
         content_url: "https://ark.stackhpc.com/pulp/content"
       register: _result
 
@@ -13,4 +13,4 @@
         backup: true
       vars:
         repo_template:
-          appliances_pulp_repos: "{{ _result.timestamps }}"
+          dnf_repos_default: "{{ _result.timestamps }}"

ansible/filter_plugins/utils.py

@@ -61,11 +61,6 @@ def to_ood_regex(items):
     r = ['(%s)' % v for v in r]
     return '|'.join(r)
 
-def appliances_repo_to_subpath(repo_entry):
-    """ Take an element from appliances_pulp_repos and convert it to a pulp path. This assumes that the remote and local pulp structures are the same
-    """
-    return repo_entry['path'] + '/' + repo_entry['timestamp']
-
 class FilterModule(object):
     ''' Ansible core jinja2 filters '''
 
@@ -81,5 +76,4 @@ def filters(self):
             'exists': exists,
             'warn': self.warn,
             'to_ood_regex': to_ood_regex,
-            'appliances_repo_to_subpath': appliances_repo_to_subpath
         }

ansible/library/latest_timestamps.py

@@ -52,18 +52,21 @@ def run_module():
     )
 
     timestamps = dict(module.params['repos_dict'])
-    for repo in timestamps:
-        for version in timestamps[repo]:
+    for index in timestamps:
+        for repo in timestamps[index]:
+            for version in timestamps[index][repo]:
 
-            html_txt = requests.get(
-                    url= module.params['content_url'] + '/' + timestamps[repo][version]['path']
-                ).text
-            timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html
-            timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags
-            latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
-            timestamps[repo][version]['timestamp'] = latest_timestamp
+                html_txt = requests.get(
+                        url= module.params['content_url'] + '/' + timestamps[index][repo][version]['pulp_path']
+                    ).text
+                timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html
+                timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags
+                latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
+                timestamps[index][repo][version]['pulp_timestamp'] = latest_timestamp
+        # timestamps[index] = dict(sorted(timestamps[index]))
 
-    result['timestamps'] = dict(sorted(timestamps.items()))
+
+    result['timestamps'] = timestamps
 
     module.exit_json(**result)
 

ansible/roles/dnf_repos/defaults/main.yml

@@ -1,54 +1,8 @@
+dnf_repos_default: {} # see environments/common/inventory/group_vars/all/timestamps.yml
+dnf_repos_extra: {}
+dnf_repos_no_epel: {}
+dnf_repos_all: {}
+dnf_repos_default_epel: {}
 dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content"
 dnf_repos_username: "{{ omit }}"
 dnf_repos_password: "{{ omit }}"
-
-dnf_repos_filenames:
-  '8':
-    baseos: 'Rocky-BaseOS'
-    appstream: 'Rocky-AppStream'
-    crb: 'Rocky-PowerTools'
-    extras: 'Rocky-Extras'
-    grafana: 'grafana'
-  '9':
-    baseos: 'rocky'
-    appstream: 'rocky'
-    crb: 'rocky'
-    extras: 'rocky-extras'
-    grafana: 'grafana'
-
-dnf_repos_version_filenames: "{{ dnf_repos_filenames[ansible_distribution_major_version] }}"
-
-# epel installed separately
-dnf_repos_default_repolist:
-- file: "{{ dnf_repos_version_filenames.baseos }}"
-  name: baseos
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.baseos[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.appstream }}"
-  name: appstream
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.appstream[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.crb }}"
-  name: "{{ 'powertools' if ansible_distribution_major_version == '8' else 'crb' }}"
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.crb[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.extras }}"
-  name: extras
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.extras[ansible_distribution_version] | appliances_repo_to_subpath }}"
-- file: ceph
-  name: Ceph
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.ceph[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-- file: "{{ dnf_repos_version_filenames.grafana }}"
-  name: grafana
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.grafana[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-
-dnf_repos_openhpc_repolist:
-- name: OpenHPC
-  file: OpenHPC
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_base[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-- name: OpenHPC-updates
-  file: OpenHPC
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_updates[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-
-dnf_repos_extra_repolist: []
-dnf_repos_repolist: "{{ dnf_repos_default_repolist + (dnf_repos_openhpc_repolist if (openhpc_install_type | default('ohpc')) == 'ohpc' else []) + dnf_repos_extra_repolist }}"
-
-dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.epel[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-dnf_repos_epel_description: "epel"

ansible/roles/dnf_repos/tasks/set_repos.yml

@@ -2,26 +2,41 @@
 
 - name: Replace system repos with Pulp repos
   ansible.builtin.yum_repository:
-    file: "{{ item.file }}"
-    name: "{{ item.name }}"
-    baseurl: "{{ item.base_url }}"
-    description: "{{ item.name }}"
+    file: "{{ repo_values.repo_file }}"
+    name: "{{ repo_name }}"
+    baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}"
+    description: "{{ repo_name }}"
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
     gpgcheck: false
-  loop: "{{ dnf_repos_repolist }}"
+  loop: "{{ dnf_repos_no_epel | dict2items }}"
+  loop_control:
+    label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
+  vars:
+    repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}"
+    repo_values: "{{ item.value[repo_os] }}"
+    repo_name: "{{ repo_values.repo_name | default(item.key) }}"
+    repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}"
 
 - name: Install epel-release
-  # done so that roles installing epel via epel-release don't over-write our changes to the epel repo
+  # So roles installing epel via epel-release don't overwrite changes to the epel repo below
   ansible.builtin.dnf:
     name: epel-release
 
-- name: Use Pulp EPEL repo
+- name: Replace epel repo with Pulp repo
   ansible.builtin.yum_repository:
-    name: epel
-    file: epel
-    description: "{{ dnf_repos_epel_description }}"
-    gpgcheck: false
-    baseurl: "{{ dnf_repos_epel_baseurl }}"
+    file: "{{ repo_values.repo_file }}"
+    name: "{{ repo_name }}"
+    baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}"
+    description: "{{ repo_name }}"
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
+    gpgcheck: false # TODO: is this really false here and above??
+  loop: "{{ dnf_repos_default_epel | dict2items }}"
+  loop_control:
+    label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
+  vars:
+    repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}"
+    repo_values: "{{ item.value[repo_os] }}"
+    repo_name: "{{ repo_values.repo_name | default(item.key) }}"
+    repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}"

ansible/roles/pulp_site/defaults/main.yml

@@ -1,44 +1,30 @@
 pulp_site_url: "{{ appliances_pulp_url }}"
 pulp_site_port: 8080
 pulp_site_username: admin # shouldn't be changed
-pulp_site_password: "{{ vault_pulp_admin_password }}"
+pulp_site_password: "{{ vault_pulp_admin_password }}" #todo make more obvious this is different from the password needed for ark (pulp_site_upstream_password)
 pulp_site_upstream_content_url: https://ark.stackhpc.com/pulp/content
 pulp_site_default_upstream_suffix: "{{ pulp_site_target_arch }}/os"
 pulp_site_validate_certs: false
 pulp_site_install_dir: '/home/rocky/pulp'
 pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}"
 pulp_site_target_facts: "{{ hostvars[groups['pulp'][0]]['ansible_facts'] }}"
-pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}"
-pulp_site_target_distribution_version_major: "{{ pulp_site_target_facts['distribution_major_version'] }}"
-
-pulp_site_rpm_info:
-- name: "baseos-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.baseos[pulp_site_target_distribution_version].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.baseos[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
-- name: "appstream-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.appstream[pulp_site_target_distribution_version].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.appstream[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
-- name: "crb-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.crb[pulp_site_target_distribution_version].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.crb[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
-- name: "extras-{{ pulp_site_target_distribution_version }}-{{ appliances_pulp_repos.extras[pulp_site_target_distribution_version].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.extras[pulp_site_target_distribution_version] | appliances_repo_to_subpath }}"
-- name: "epel-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.epel[pulp_site_target_distribution_version_major].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.epel[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
-- name: "ohpc-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.openhpc_base[pulp_site_target_distribution_version_major].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.openhpc_base[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
-- name: "ohpc-updates-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.openhpc_updates[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
-- name: "ceph-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major].timestamp }}"
-  subpath: "{{ appliances_pulp_repos.ceph[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
-- name: "grafana-{{ pulp_site_target_distribution_version_major }}-{{ appliances_pulp_repos.grafana.timestamp[pulp_site_target_distribution_version_major].timestamp }}
-  subpath: "{{ appliances_pulp_repos.grafana[pulp_site_target_distribution_version_major] | appliances_repo_to_subpath }}"
+pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}" # TODO: how to set automatically?
 
 pulp_site_rpm_repo_defaults:
   remote_username: "{{ pulp_site_upstream_username }}"
   remote_password: "{{ pulp_site_upstream_password }}"
   policy: on_demand
-  state: present
 
-_pulp_site_rpm_info_all: "{{ pulp_site_rpm_info | map('combine', pulp_site_rpm_repo_defaults) }}"
+pulp_site_rpm_info: |
+  {{
+    dnf_repos_all |
+    select_repos(pulp_site_target_distribution_version)
+  }}
+pulp_site_rpm_repos: |
+  {{
+    pulp_site_rpm_info |
+    to_rpm_repos(pulp_site_upstream_content_url, pulp_site_rpm_repo_defaults)
+  }}
 
-pulp_site_rpm_repos: "{{ _pulp_site_rpm_info_all | to_rpm_repos(pulp_site_upstream_content_url) }}"
-pulp_site_rpm_publications: "{{ _pulp_site_rpm_info_all | to_rpm_pubs }}"
-pulp_site_rpm_distributions: "{{ _pulp_site_rpm_info_all | to_rpm_distros }}"
+pulp_site_rpm_publications: "{{ pulp_site_rpm_info | to_rpm_pubs }}"
+pulp_site_rpm_distributions: "{{ pulp_site_rpm_info | to_rpm_distros }}"

ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py

@@ -3,29 +3,56 @@ def filters(self):
         return {
             'to_rpm_repos': self.to_rpm_repos,
             'to_rpm_pubs': self.to_rpm_pubs,
-            'to_rpm_distros': self.to_rpm_distros
+            'to_rpm_distros': self.to_rpm_distros,
+            'select_repos': self.select_repos,
         }
-
-    def to_rpm_repos(self, list, pulp_url):
-        repo_list = map(lambda x: {
-            'name': x['name'],
-            'url': pulp_url+'/'+x['subpath'],
-            'remote_username': x['remote_username'],
-            'remote_password': x['remote_password'],
-            'policy': x['policy'],
-            'state': x['state'] }, list)
-        return repo_list
 
+    def select_repos(self, dnf_repos, target_distro_ver): #TODO: why does baseos get a major and minor version?
+        """ Filter dnf_repos to only those for a relevant distribution version (M.m or M). Returns a list of dicts.
+            TODO: note this adds distro_ver and pulp_repo_name as a key
+        """
+
+        target_distro_ver_major = target_distro_ver.split('.')[0]
+
+        rpm_repos = []
+        for repokey in dnf_repos:
+            # select either the matching major.minor or major version:
+            if target_distro_ver in dnf_repos[repokey]:
+                selected_ver = target_distro_ver
+            elif target_distro_ver_major in dnf_repos[repokey]:
+                selected_ver = target_distro_ver_major
+            else:
+                raise ValueError(f'No key matching {target_distro_ver_major} or {target_distro_ver} found in f{repokey}')
+            repo_data = dnf_repos[repokey][selected_ver]
+            repo_data['distro_ver'] = selected_ver
+            repo_data['pulp_repo_name'] = repokey
+            rpm_repos.append(repo_data)
+        return rpm_repos
+
+    def to_rpm_repos(self, rpm_info, content_url, repo_defaults):
+        """ TODO """
+        rpm_repos = []
+        for repo_data in rpm_info:
+            rpm_data = repo_defaults.copy() # NB: this changes behaviour vs before, so now defaults can correctly be overriden
+            rpm_data['name'] = get_repo_name(repo_data)
+            rpm_data['url'] = '/'.join([content_url, repo_data['pulp_path'], repo_data['pulp_timestamp']])
+            rpm_data['state'] = 'present'
+            rpm_repos.append(rpm_data)
+        return rpm_repos
+
     def to_rpm_pubs(self, list):
         pub_list = map(lambda x: {
-            'repository': x['name'],
-            'state': x['state'] }, list)
+            'repository': get_repo_name(x),
+            'state': 'present' }, list)
         return pub_list
 
     def to_rpm_distros(self, list):
         distro_list = map(lambda x: {
-            'name': x['name'],
-            'repository': x['name'],
-            'base_path': x['subpath'],
-            'state': x['state'] }, list)
-        return distro_list
+            'name': x['pulp_repo_name'],
+            'repository': get_repo_name(x),
+            'base_path': '/'.join([x['pulp_path'],x['pulp_timestamp']]),
+            'state': 'present' }, list)
+        return distro_list
+
+def get_repo_name(dnf_repos_data):
+    return f"{dnf_repos_data['pulp_repo_name']}-{dnf_repos_data['distro_ver']}-{dnf_repos_data['pulp_timestamp']}"

ansible/roles/pulp_site/tasks/sync.yml

@@ -3,17 +3,7 @@
 - ansible.builtin.assert:
     that: pulp_site_upstream_password != ''
     quiet: true
-    fail_msg: "Upstream password not set. Either set env var ARK_PASSWORD or override pulp_site_upstream_password."
-
-- name: Wait for Pulp server
-  pulp.squeezer.status:
-    pulp_url: "{{ pulp_site_url }}"
-    username: "{{ pulp_site_username }}"
-    password: "{{ pulp_site_password }}"
-  register: _pulp_status
-  until: _pulp_status.failed == false
-  retries: 30
-  delay: 20
+    fail_msg: "Upstream password not set. Ensure `pulp_site_upstream_username` and `pulp_site_upstream_password` are overriden to your Ark credentials."
 
 - name: Ensure Pulp CLI config directory exists
   ansible.builtin.file:
@@ -27,6 +17,16 @@
     dest: ~/.config/pulp/cli.toml
     mode: '0644'
 
+- name: Wait for Pulp server
+  pulp.squeezer.status:
+    pulp_url: "{{ pulp_site_url }}"
+    username: "{{ pulp_site_username }}"
+    password: "{{ pulp_site_password }}"
+  register: _pulp_status
+  until: _pulp_status.failed == false
+  retries: 30
+  delay: 20
+
 - block:
   - name: Ensure squeezer cache exists
     ansible.builtin.file:

ansible/roles/pulp_site/templates/cli.toml.j2

@@ -4,7 +4,6 @@ username = "{{ pulp_site_username }}"
 password = "{{ pulp_site_password }}"
 api_root = "/pulp/"
 domain = "default"
-headers = []
 cert = ""
 key = ""
 verify_ssl = true