Merge remote-tracking branch 'origin/main' into ci/enable-linting

Author: maxstack

ansible/adhoc/deploy-pulp.yml

@@ -1,27 +1,18 @@
 ---
-# Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<pulp server hostname>"
-
-- name: Add temporary pulp server host
-  hosts: localhost
-  tasks:
-    - ansible.builtin.add_host:
-        name: "{{ pulp_server }}"
-        group: "_pulp_host"
-
-- name: Install pulp on server and add to config
-  become: true
-  hosts: _pulp_host
+- name: Install pulp on server
+  become: yes
+  hosts: pulp_server
   tasks:
     - name: Install pulp
       ansible.builtin.include_role:
         name: pulp_site
         tasks_from: install.yml
         public: true
 
-    - name: Print Pulp endpoint
-      become: false
-      ansible.builtin.debug:
-        msg: |
-          Server configured, override 'appliances_pulp_url' with
-            appliances_pulp_url: "http://{{ pulp_server }}:{{ pulp_site_port }}"
-          in your environments
+  - name: Print Pulp endpoint
+    become: no
+    debug:
+      msg: | 
+        Server configured, override 'appliances_pulp_url' with
+          appliances_pulp_url: "http://{{ hostvars[groups['pulp_server'] | first].ansible_host }}:{{ pulp_site_port }}"
+        (or the correct IP if multi-homed) in your environments

ansible/adhoc/sync-pulp.yml

@@ -7,5 +7,5 @@
       vars:
         pulp_site_target_arch: "x86_64"
         pulp_site_target_distribution: "rocky"
-        pulp_site_target_distribution_version: "9.5"
-        pulp_site_target_distribution_version_major: "9"
+        # default distribution to *latest* specified for baseos repo:
+        pulp_site_target_distribution_version: "{{ dnf_repos_repos['baseos'].keys() | map('float') | sort | last }}"

ansible/ci/update_timestamps.yml

@@ -3,15 +3,15 @@
   tasks:
     - name: Get latest timestamps from sources # noqa: syntax-check[unknown-module] # ansible/library/latest_timestamps.py
       latest_timestamps:
-        repos_dict: "{{ appliances_pulp_repos }}"
+        repos_dict: "{{ dnf_repos_default }}"
         content_url: "https://ark.stackhpc.com/pulp/content"
       register: _result
 
     - name: Overwrite repo timestamps with latest
       ansible.builtin.copy:
-        dest: "{{ appliances_repository_root }}/environments/common/inventory/group_vars/all/timestamps.yml"
+        dest: "{{ appliances_repository_root }}/environments/common/inventory/group_vars/all/dnf_repo_timestamps.yml"
         content: "{{ repo_template | to_nice_yaml(indent=2) }}"
         backup: true
       vars:
         repo_template:
-          appliances_pulp_repos: "{{ _result.timestamps }}"
+          dnf_repos_default: "{{ _result.timestamps }}"

ansible/fatimage.yml

@@ -19,7 +19,7 @@
   when: hook_path | exists
 
 - name: Sync pulp repos with upstream
-  hosts: pulp
+  hosts: pulp_site
   tasks:
     - ansible.builtin.include_role:
         name: pulp_site

ansible/filter_plugins/utils.py

@@ -56,21 +56,13 @@ def to_ood_regex(items):
 
     # There's a python bug which means re.sub() can't use '\d' in the replacement so
     # have to do replacement in two stages:
-    r = [re.sub(r"\d+", "XBACKSLASHX", v) for v in items]
-    r = [v.replace("XBACKSLASHX", "\\d+") for v in set(r)]
-    r = ["(%s)" % v for v in r]  # pylint: disable=consider-using-f-string
-    return "|".join(r)
+    r = [re.sub(r"\d+", 'XBACKSLASHX', v) for v in items]
+    r = [v.replace('XBACKSLASHX', '\d+') for v in set(r)]
+    r = ['(%s)' % v for v in r]
+    return '|'.join(r)
 
-
-def appliances_repo_to_subpath(repo_entry):
-    """Take an element from appliances_pulp_repos and convert it to a pulp path.
-    This assumes that the remote and local pulp structures are the same
-    """
-    return repo_entry["path"] + "/" + repo_entry["timestamp"]
-
-
-class FilterModule(object):  # pylint: disable=useless-object-inheritance
-    """Ansible core jinja2 filters"""
+class FilterModule(object):
+    ''' Ansible core jinja2 filters '''
 
     # pylint: disable=missing-function-docstring
     def warn(self, message, **kwargs):  # pylint: disable=unused-argument
@@ -80,10 +72,9 @@ def warn(self, message, **kwargs):  # pylint: disable=unused-argument
     def filters(self):  # pylint: disable=missing-function-docstring
         return {
             # jinja2 overrides
-            "readfile": readfile,
-            "prometheus_node_exporter_targets": prometheus_node_exporter_targets,
-            "exists": exists,
-            "warn": self.warn,
-            "to_ood_regex": to_ood_regex,
-            "appliances_repo_to_subpath": appliances_repo_to_subpath,
+            'readfile': readfile,
+            'prometheus_node_exporter_targets': prometheus_node_exporter_targets,
+            'exists': exists,
+            'warn': self.warn,
+            'to_ood_regex': to_ood_regex,
         }

ansible/library/latest_timestamps.py

@@ -63,24 +63,13 @@ def run_module():  # pylint: disable=missing-function-docstring
         for version in timestamps[repo]:
 
             html_txt = requests.get(
-                url=module.params["content_url"]
-                + "/"
-                + timestamps[repo][version]["path"]
-            ).text
-            timestamp_link_list = (
-                BeautifulSoup(html_txt, features="html.parser")
-                .body.find("pre")
-                .find_all()
-            )  # getting raw list of timestamps from html
-            timestamp_link_list = map(
-                lambda x: x.string, timestamp_link_list
-            )  # stripping xml tags
-            latest_timestamp = list(timestamp_link_list)[-1][
-                :-1
-            ]  # last timestamp in list with trailing / removed
-            timestamps[repo][version]["timestamp"] = latest_timestamp
-
-    result["timestamps"] = dict(sorted(timestamps.items()))
+                    url= module.params['content_url'] + '/' + timestamps[repo][version]['pulp_path']
+                ).text
+            timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html
+            timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags
+            latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
+            timestamps[repo][version]['pulp_timestamp'] = latest_timestamp
+    result['timestamps'] = dict(sorted(timestamps.items()))
 
     module.exit_json(**result)
 

ansible/roles/cuda/defaults/main.yml

@@ -1,9 +1,9 @@
 ---
 # yamllint disable-line rule:line-length
 cuda_repo_url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
-cuda_nvidia_driver_stream: "575-open"
-cuda_nvidia_driver_pkg: "nvidia-open-3:575.57.08-1.el{{ ansible_distribution_major_version }}"
-cuda_package_version: "12.9.1-1"
+cuda_nvidia_driver_stream: '580-open'
+cuda_nvidia_driver_pkg: "nvidia-open-3:580.65.06-1.el{{ ansible_distribution_major_version }}"
+cuda_package_version: '13.0.0-1'
 cuda_version_short: "{{ (cuda_package_version | split('.'))[0:2] | join('.') }}" # major.minor
 cuda_packages:
   - "cuda-toolkit-{{ cuda_package_version }}"

ansible/roles/dnf_repos/README.md

@@ -0,0 +1,40 @@
+dnf_repos
+=========
+
+Modifies repo definitions for repofiles in `/etc/yum.repos.d` to point to snapshots in StackHPC's Ark Pulp server or mirrors of them
+on a local Pulp server.
+
+Requirements
+------------
+
+Requires Ark credentials if using StackHPC's upstream Ark server.
+
+Role Variables
+--------------
+
+Variables in this role are also required by `pulp_site` so set in 
+`environments/common/inventory/groups_vars/all/dnf_repos.yml`. See that file for detailed default values.
+
+- `dnf_repos_repos`: Dict of dicts containing information to construct URLs for Ark snapshots from the target Pulp server for each Rocky version. For example:
+    ```
+    dnf_repos_repos:
+        appstream:                          # ansible.builtin.yum_repository:name
+            '8.10':                           # ansible_distribution_version or ansible_distribution_major_version
+                repo_file: Rocky-AppStream      # yum_repository: file
+                # repo_name:                    # optional, override yum_repository:name
+                pulp_path: rocky/8.10/AppStream/x86_64/os # The subpath of the the upstream Ark server's content endpoint URL for the repo's snapshots, see https://ark.stackhpc.com/pulp/content/
+                pulp_timestamp: 20250614T013846
+                # pulp_content_url:             # optional, dnf_repos_pulp_content_url
+            '9.6':
+                ...
+    ```
+- `dnf_repos_default`: Appliance default repos to use Ark snapshots for. Following same format as `dnf_repos_repos`.
+  See for appliance default repo list `environments/common/inventory/group_vars/all/dnf_repo_timestamps.yml`.
+- `dnf_repos_extra`: Additional repos to use Ark snapshots for. Follows same format as
+  `dnf_repos_repos`. Defaults to `{}`
+- `dnf_repos_pulp_content_url`: Optional str. Content URL of Pulp server to use Ark snapshots from. 
+  Defaults to `{{ appliances_pulp_url }}/pulp/content`
+- `dnf_repos_username`: Optional str. Username for Ark. Should be set if using upstream StackHPC Ark
+  Pulp server, but omitted if using local Pulp server (see `ansible/roles/pulp_site`)
+- `dnf_repos_password`: Optional str. Password for Ark. Should be set if using upstream StackHPC Ark
+  Pulp server, but omitted if using local Pulp server (see `ansible/roles/pulp_site`)

ansible/roles/dnf_repos/defaults/main.yml

@@ -1,55 +1,5 @@
 ---
+dnf_repos_repos: {} # see environments/common/inventory/group_vars/all/{dnf_repos,timestamps}.yml
 dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content"
 dnf_repos_username: "{{ omit }}"
 dnf_repos_password: "{{ omit }}"
-
-dnf_repos_filenames:
-  "8":
-    baseos: "Rocky-BaseOS"
-    appstream: "Rocky-AppStream"
-    crb: "Rocky-PowerTools"
-    extras: "Rocky-Extras"
-    grafana: "grafana"
-  "9":
-    baseos: "rocky"
-    appstream: "rocky"
-    crb: "rocky"
-    extras: "rocky-extras"
-    grafana: "grafana"
-
-dnf_repos_version_filenames: "{{ dnf_repos_filenames[ansible_distribution_major_version] }}"
-
-# epel installed separately
-dnf_repos_default_repolist:
-  - file: "{{ dnf_repos_version_filenames.baseos }}"
-    name: baseos
-    base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.baseos[ansible_distribution_version] | appliances_repo_to_subpath }}"
-  - file: "{{ dnf_repos_version_filenames.appstream }}"
-    name: appstream
-    base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.appstream[ansible_distribution_version] | appliances_repo_to_subpath }}"
-  - file: "{{ dnf_repos_version_filenames.crb }}"
-    name: "{{ 'powertools' if ansible_distribution_major_version == '8' else 'crb' }}"
-    base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.crb[ansible_distribution_version] | appliances_repo_to_subpath }}"
-  - file: "{{ dnf_repos_version_filenames.extras }}"
-    name: extras
-    base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.extras[ansible_distribution_version] | appliances_repo_to_subpath }}"
-  - file: ceph
-    name: Ceph
-    base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.ceph[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-  - file: "{{ dnf_repos_version_filenames.grafana }}"
-    name: grafana
-    base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.grafana[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-
-dnf_repos_openhpc_repolist:
-  - name: OpenHPC
-    file: OpenHPC
-    base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_base[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-  - name: OpenHPC-updates
-    file: OpenHPC
-    base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_updates[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-
-dnf_repos_extra_repolist: []
-dnf_repos_repolist: "{{ dnf_repos_default_repolist + (dnf_repos_openhpc_repolist if (openhpc_install_type | default('ohpc')) == 'ohpc' else []) + dnf_repos_extra_repolist }}" # noqa: yaml[line-length]
-
-dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.epel[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
-dnf_repos_epel_description: "epel"

ansible/roles/dnf_repos/tasks/disable_repos.yml

@@ -1,21 +1,20 @@
 ---
 - name: Remove password and disable Pulp repos
   ansible.builtin.yum_repository:
-    file: "{{ item.file }}"
-    name: "{{ item.name }}"
-    baseurl: "{{ item.base_url }}"
-    description: "{{ item.name }}"
+    file: "{{ repo_values.repo_file }}"
+    name: "{{ repo_name }}"
+    baseurl: "{{ repo_content_url }}/{{ repo_values.pulp_path }}/{{ repo_values.pulp_timestamp }}"
+    description: "{{ repo_name }}"
     enabled: false
-  loop: "{{ dnf_repos_repolist }}"
-
-- name: Remove password and disable EPEL repo
-  ansible.builtin.yum_repository:
-    name: epel
-    file: epel
-    description: "{{ dnf_repos_epel_description }}"
-    baseurl: "{{ dnf_repos_epel_baseurl }}"
     gpgcheck: false
-    enabled: false
+  loop: "{{ dnf_repos_repos | dict2items }}"
+  loop_control:
+    label: "{{ repo_name }}[{{ repo_os }}]: {{ repo_values }}"
+  vars:
+    repo_os: "{{ ansible_distribution_version if ansible_distribution_version in item.value else ansible_distribution_major_version }}"
+    repo_values: "{{ item.value[repo_os] }}"
+    repo_name: "{{ repo_values.repo_name | default(item.key) }}"
+    repo_content_url: "{{ repo_values.pulp_content_url | default(dnf_repos_pulp_content_url) }}"
 
 - name: Get all repo files
   ansible.builtin.find: