Fix linting on merge of origin/main

Author: maxstack

ansible/filter_plugins/utils.py

@@ -56,25 +56,28 @@ def to_ood_regex(items):
 
     # There's a python bug which means re.sub() can't use '\d' in the replacement so
     # have to do replacement in two stages:
-    r = [re.sub(r"\d+", 'XBACKSLASHX', v) for v in items]
-    r = [v.replace('XBACKSLASHX', '\d+') for v in set(r)]
-    r = ['(%s)' % v for v in r]
-    return '|'.join(r)
+    r = [re.sub(r"\d+", "XBACKSLASHX", v) for v in items]
+    r = [v.replace("XBACKSLASHX", r"\d+") for v in set(r)]
+    r = [f"({v})" for v in r]
+    return "|".join(r)
 
+
+# pylint: disable=useless-object-inheritance
 class FilterModule(object):
-    ''' Ansible core jinja2 filters '''
+    """Ansible core jinja2 filters"""
 
     # pylint: disable=missing-function-docstring
     def warn(self, message, **kwargs):  # pylint: disable=unused-argument
         Display().warning(message)
         return message
 
-    def filters(self):  # pylint: disable=missing-function-docstring
+    # pylint: disable=missing-function-docstring
+    def filters(self):
         return {
             # jinja2 overrides
-            'readfile': readfile,
-            'prometheus_node_exporter_targets': prometheus_node_exporter_targets,
-            'exists': exists,
-            'warn': self.warn,
-            'to_ood_regex': to_ood_regex,
+            "readfile": readfile,
+            "prometheus_node_exporter_targets": prometheus_node_exporter_targets,
+            "exists": exists,
+            "warn": self.warn,
+            "to_ood_regex": to_ood_regex,
         }

ansible/library/latest_timestamps.py

@@ -63,13 +63,23 @@ def run_module():  # pylint: disable=missing-function-docstring
         for version in timestamps[repo]:
 
             html_txt = requests.get(
-                    url= module.params['content_url'] + '/' + timestamps[repo][version]['pulp_path']
-                ).text
-            timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html
-            timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags
-            latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
-            timestamps[repo][version]['pulp_timestamp'] = latest_timestamp
-    result['timestamps'] = dict(sorted(timestamps.items()))
+                url=module.params["content_url"]
+                + "/"
+                + timestamps[repo][version]["pulp_path"]
+            ).text
+            timestamp_link_list = (
+                BeautifulSoup(html_txt, features="html.parser")
+                .body.find("pre")
+                .find_all()
+            )  # getting raw list of timestamps from html
+            timestamp_link_list = map(
+                lambda x: x.string, timestamp_link_list
+            )  # stripping xml tags
+            latest_timestamp = list(timestamp_link_list)[-1][
+                :-1
+            ]  # last timestamp in list with trailing / removed
+            timestamps[repo][version]["pulp_timestamp"] = latest_timestamp
+    result["timestamps"] = dict(sorted(timestamps.items()))
 
     module.exit_json(**result)
 

ansible/roles/dnf_repos/README.md

@@ -1,38 +1,34 @@
-dnf_repos
-=========
+# dnf_repos
 
-Modifies repo definitions for repofiles in `/etc/yum.repos.d` to point to snapshots in StackHPC's Ark Pulp server or mirrors of them
+Modifies repository definitions for repofiles in `/etc/yum.repos.d` to point to snapshots in StackHPC's Ark Pulp server or mirrors of them
 on a local Pulp server.
 
-Requirements
-------------
+## Requirements
 
 Requires Ark credentials if using StackHPC's upstream Ark server.
 
-Role Variables
---------------
+## Role Variables
 
-Variables in this role are also required by `pulp_site` so set in 
+Variables in this role are also required by `pulp_site` so set in
 `environments/common/inventory/groups_vars/all/dnf_repos.yml`. See that file for detailed default values.
 
 - `dnf_repos_repos`: Dict of dicts containing information to construct URLs for Ark snapshots from the target Pulp server for each Rocky version. For example:
-    ```
-    dnf_repos_repos:
-        appstream:                          # ansible.builtin.yum_repository:name
-            '8.10':                           # ansible_distribution_version or ansible_distribution_major_version
-                repo_file: Rocky-AppStream      # yum_repository: file
-                # repo_name:                    # optional, override yum_repository:name
-                pulp_path: rocky/8.10/AppStream/x86_64/os # The subpath of the the upstream Ark server's content endpoint URL for the repo's snapshots, see https://ark.stackhpc.com/pulp/content/
-                pulp_timestamp: 20250614T013846
-                # pulp_content_url:             # optional, dnf_repos_pulp_content_url
-            '9.6':
-                ...
-    ```
+  ```yaml
+  dnf_repos_repos:
+    appstream: # ansible.builtin.yum_repository:name
+      "8.10": # ansible_distribution_version or ansible_distribution_major_version
+        repo_file: Rocky-AppStream # yum_repository: file
+        # repo_name:                    # optional, override yum_repository:name
+        pulp_path: rocky/8.10/AppStream/x86_64/os # The subpath of the the upstream Ark server's content endpoint URL for the repo's snapshots, see https://ark.stackhpc.com/pulp/content/
+        pulp_timestamp: 20250614T013846
+        # pulp_content_url:             # optional, dnf_repos_pulp_content_url
+      "9.6": ...
+  ```
 - `dnf_repos_default`: Appliance default repos to use Ark snapshots for. Following same format as `dnf_repos_repos`.
-  See for appliance default repo list `environments/common/inventory/group_vars/all/dnf_repo_timestamps.yml`.
+  See for appliance default repository list `environments/common/inventory/group_vars/all/dnf_repo_timestamps.yml`.
 - `dnf_repos_extra`: Additional repos to use Ark snapshots for. Follows same format as
   `dnf_repos_repos`. Defaults to `{}`
-- `dnf_repos_pulp_content_url`: Optional str. Content URL of Pulp server to use Ark snapshots from. 
+- `dnf_repos_pulp_content_url`: Optional str. Content URL of Pulp server to use Ark snapshots from.
   Defaults to `{{ appliances_pulp_url }}/pulp/content`
 - `dnf_repos_username`: Optional str. Username for Ark. Should be set if using upstream StackHPC Ark
   Pulp server, but omitted if using local Pulp server (see `ansible/roles/pulp_site`)

ansible/roles/hpctests/library/plot_nxnlatbw.py

@@ -178,7 +178,7 @@ def run_module():  # pylint: disable=missing-function-docstring, too-many-locals
                     float(vals[3]),
                 )
             except ValueError:
-                print('warning: skipping line %i (%s) - parse failure' % (ln, line))
+                print(f"warning: skipping line {ln} ({line}) - parse failure")
                 continue
             latencies[rankA, rankB] = lat
             bandwidths[rankA, rankB] = bw
@@ -193,7 +193,7 @@ def run_module():  # pylint: disable=missing-function-docstring, too-many-locals
         module.fail_json(
             "Results contained %i ranks but %i node names provided"  # pylint: disable=consider-using-f-string
             % (len(rankAs), len(nodes)),
-            **result
+            **result,
         )
 
     # find min values:

ansible/roles/pulp_site/README.md

@@ -1,21 +1,18 @@
-pulp_site
-=========
+# pulp_site
 
-Contains playbooks to deploy a Pulp server and sync its content with repo snapshots in
+Contains playbooks to deploy a Pulp server and sync its content with repository snapshots in
 StackHPC's Ark Pulp server
 
-Requirements
-------------
+## Requirements
 
 Requires Ark credentials. The VM you are deploying Pulp on must allow ingress on `pulp_site_port`
 and not be externally accessible (as the Pulp server's content is unauthenticated). Rocky Linux 9 has been
 tested as the target VM for deploying Pulp.
 
-Role Variables
---------------
+## Role Variables
 
-- `pulp_site_url`: Required str. The base url from which Pulp content will be hosted. Defaults to `{{ appliances_pulp_url }}`. 
-                 Value to set for ``appliances_pulp_url` will be generated and output by the deploy.yml playbook.
+- `pulp_site_url`: Required str. The base URL from which Pulp content will be hosted. Defaults to `{{ appliances_pulp_url }}`.
+  Value to set for ``appliances_pulp_url` will be generated and output by the deploy.yml playbook.
 - `pulp_site_port`: Optional str. Port to serve Pulp server on. Defaults to `8080`.
 - `pulp_site_username`: Optional str. Admin username for the Pulp server. Defaults to `admin`.
 - `pulp_site_password`: Required str. Admin password for the Pulp server. Defaults to `{{ vault_pulp_admin_password }}`.
@@ -24,13 +21,13 @@ Role Variables
 - `pulp_site_upstream_content_url`: Optional str. Content URL of upstream Ark Pulp. Defaults to `https://ark.stackhpc.com/pulp/content`.
 - `pulp_site_install_dir`: Optional str. Directory on Pulp host to install config and persistent state to be mounted into Pulp container. Defaults to `/home/rocky/pulp`.
 - `pulp_site_target_facts`: Optional str. The `ansible_facts` of a host which will be pulling from your Pulp server, allowing the role to auto-discover the necessary repos to pull.
-                          defaults to `{{ hostvars[groups['pulp'][0]]['ansible_facts'] }}`.
+  defaults to `{{ hostvars[groups['pulp'][0]]['ansible_facts'] }}`.
 - `pulp_site_target_distribution_version`: Optional str. The Rocky Linux minor release to sync repos from Ark for. Defaults to `{{ pulp_site_target_facts['distribution_version'] }}`.
-- `pulp_site_rpm_repo_defaults`: Optional dict. Contains key value pairs for fields which are common to all repo definition in `pulp_site_rpm_repos`. Includes values for `remote_username`,
-                               `remote_password` and `policy` by default.
-- `pulp_site_rpm_repos`: Optional list of dicts. List of repo definitions in format required by the `stackhpc.pulp.pulp_repository`. Defaults to modified versions of repos defined in
-                       `dnf_repos_all`.
-- `pulp_site_rpm_publications`: Optional list of dicts. List of repo definitions in format required by the `stackhpc.pulp.pulp_publication`. Defaults to list of publications for repos defined in
-                              `dnf_repos_all`.
-- `pulp_site_rpm_distributions`: Optional list of dicts. List of repo definitions in format required by the `stackhpc.pulp.pulp_distribution`. Defaults to list of distributions for repos defined in
-                              `dnf_repos_all`.
+- `pulp_site_rpm_repo_defaults`: Optional dict. Contains key-value pairs for fields which are common to all repository definition in `pulp_site_rpm_repos`. Includes values for `remote_username`,
+  `remote_password` and `policy` by default.
+- `pulp_site_rpm_repos`: Optional list of dicts. List of repository definitions in format required by the `stackhpc.pulp.pulp_repository`. Defaults to modified versions of repos defined in
+  `dnf_repos_all`.
+- `pulp_site_rpm_publications`: Optional list of dicts. List of repository definitions in format required by the `stackhpc.pulp.pulp_publication`. Defaults to list of publications for repos defined in
+  `dnf_repos_all`.
+- `pulp_site_rpm_distributions`: Optional list of dicts. List of repository definitions in format required by the `stackhpc.pulp.pulp_distribution`. Defaults to list of distributions for repos defined in
+  `dnf_repos_all`.

ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py

@@ -4,19 +4,20 @@ class FilterModule(object):
 
     def filters(self):  # pylint: disable=missing-function-docstring
         return {
-            'to_rpm_repos': self.to_rpm_repos,
-            'to_rpm_pubs': self.to_rpm_pubs,
-            'to_rpm_distros': self.to_rpm_distros,
-            'select_repos': self.select_repos,
+            "to_rpm_repos": self.to_rpm_repos,
+            "to_rpm_pubs": self.to_rpm_pubs,
+            "to_rpm_distros": self.to_rpm_distros,
+            "select_repos": self.select_repos,
         }
-    
+
     def select_repos(self, dnf_repos, target_distro_ver):
-        """ Filter dnf_repos to only those for a relevant distribution version (M.m or M). Returns a list of dicts.
-            Also adds pulp_repo_name field to give the repository a unique name in Pulp to be referenced by subsequent
-            filters
+        """Filter dnf_repos to only those for a relevant distribution version (M.m or M).
+        Returns a list of dicts.
+        Also adds pulp_repo_name field to give the repository a unique name in Pulp
+        to be referenced by subsequent filters
         """
-    
-        target_distro_ver_major = target_distro_ver.split('.')[0]
+
+        target_distro_ver_major = target_distro_ver.split(".")[0]
 
         rpm_repos = []
         for repokey in dnf_repos:
@@ -26,41 +27,54 @@ def select_repos(self, dnf_repos, target_distro_ver):
             elif target_distro_ver_major in dnf_repos[repokey]:
                 selected_ver = target_distro_ver_major
             else:
-                raise ValueError(f'No key matching {target_distro_ver_major} or {target_distro_ver} found in f{repokey}')
+                raise ValueError(
+                    # pylint: disable-next=line-too-long
+                    f"No key matching {target_distro_ver_major} or {target_distro_ver} found in f{repokey}"
+                )
             repo_data = dnf_repos[repokey][selected_ver]
-            repo_data['pulp_repo_name'] = f"{repokey}-{selected_ver}-{dnf_repos[repokey][selected_ver]['pulp_timestamp']}"
+            repo_data["pulp_repo_name"] = (
+                f"{repokey}-{selected_ver}-{dnf_repos[repokey][selected_ver]['pulp_timestamp']}"
+            )
             rpm_repos.append(repo_data)
         return rpm_repos
 
     def to_rpm_repos(self, rpm_info, content_url, repo_defaults):
-        """ Filter repo object list given by select_repos into dict required by the pulp_repository_rpm_repos variable
-            from stackhpc.pulp.pulp_repository role
+        """Filter repo object list given by select_repos into dict required by the
+        pulp_repository_rpm_repos variable from stackhpc.pulp.pulp_repository role
         """
         rpm_repos = []
         for repo_data in rpm_info:
-            rpm_data = repo_defaults.copy() # NB: this changes behaviour vs before, so now defaults can correctly be overriden
-            rpm_data['name'] = repo_data['pulp_repo_name']
-            rpm_data['url'] = '/'.join([content_url, repo_data['pulp_path'], repo_data['pulp_timestamp']])
-            rpm_data['state'] = 'present'
+            rpm_data = (
+                repo_defaults.copy()
+            )  # NB: this changes behaviour vs before, so now defaults can correctly be overriden
+            rpm_data["name"] = repo_data["pulp_repo_name"]
+            rpm_data["url"] = "/".join(
+                [content_url, repo_data["pulp_path"], repo_data["pulp_timestamp"]]
+            )
+            rpm_data["state"] = "present"
             rpm_repos.append(rpm_data)
         return rpm_repos
 
-    def to_rpm_pubs(self, list):
-        """ Filter repo object list given by select_repos into dict required by the pulp_publication_rpm variable
-            from stackhpc.pulp.pulp_publication role
+    def to_rpm_pubs(self, _list):
+        """Filter repo object list given by select_repos into dict required by the
+        pulp_publication_rpm variable from stackhpc.pulp.pulp_publication role
         """
-        pub_list = map(lambda x: {
-            'repository': x['pulp_repo_name'],
-            'state': 'present' }, list)
+        pub_list = map(
+            lambda x: {"repository": x["pulp_repo_name"], "state": "present"}, _list
+        )
         return pub_list
-    
-    def to_rpm_distros(self, list):
-        """ Filter repo object list given by select_repos into dict required by the pulp_distirubtion_rpm variable
-            from stackhpc.pulp.pulp_distribution role
+
+    def to_rpm_distros(self, _list):
+        """Filter repo object list given by select_repos into dict required by the
+        pulp_distirubtion_rpm variable from stackhpc.pulp.pulp_distribution role
         """
-        distro_list = map(lambda x: {
-            'name': x['pulp_repo_name'],
-            'repository': x['pulp_repo_name'],
-            'base_path': '/'.join([x['pulp_path'],x['pulp_timestamp']]),
-            'state': 'present' }, list)
+        distro_list = map(
+            lambda x: {
+                "name": x["pulp_repo_name"],
+                "repository": x["pulp_repo_name"],
+                "base_path": "/".join([x["pulp_path"], x["pulp_timestamp"]]),
+                "state": "present",
+            },
+            _list,
+        )
         return distro_list

docs/environments.md

@@ -14,11 +14,12 @@ All environments load the inventory from the `common` environment first, with th
 ### Environment-specific inventory structure
 
 The ansible inventory for the environment is in `environments/<environment>/inventory/`. It should generally contain:
-- A `hosts` file. This defines the hosts in the appliance. Generally it should be templated out by the deployment automation so it is also a convenient place to define variables which depend on the deployed hosts such as connection variables, IP addresses, ssh proxy arguments etc.
+
+- A `hosts` file. This defines the hosts in the appliance. Generally it should be templated out by the deployment automation so it is also a convenient place to define variables which depend on the deployed hosts such as connection variables, IP addresses, SSH proxy arguments etc.
 - A `groups` file defining ansible groups, which essentially controls which features of the appliance are enabled and where they are deployed. This repository generally follows a convention where functionality is defined using ansible roles applied to a group
-of the same name, e.g. `openhpc` or `grafana`. The meaning and use of each group is described in comments in `environments/common/inventory/groups`. As the groups defined there for the common environment are empty, functionality is disabled by default and must be
-enabled in a specific environment's `groups` file. The `site` environment contains an ini file at `environments/site/inventory/groups` which enables groups for default appliance functionality across all environments. Additional groups should generally also be
-enabled in this file to avoid divergence between staging and production environments. Note that enabling some groups may require a site-specific image build and Ark credentials (see [operations guide](operations.md)).
+  of the same name, e.g. `openhpc` or `grafana`. The meaning and use of each group is described in comments in `environments/common/inventory/groups`. As the groups defined there for the common environment are empty, functionality is disabled by default and must be
+  enabled in a specific environment's `groups` file. The `site` environment contains an ini file at `environments/site/inventory/groups` which enables groups for default appliance functionality across all environments. Additional groups should generally also be
+  enabled in this file to avoid divergence between staging and production environments. Note that enabling some groups may require a site-specific image build and Ark credentials (see [operations guide](operations.md)).
 - Optionally, group variable files in `group_vars/<group_name>/overrides.yml`, where the group names match the functional groups described above. These can be used to override the default configuration for each functionality, which are defined in `environments/common/inventory/group_vars/all/<group_name>.yml` (the use of `all` here is due to ansible's precedence rules).
 
 Although most of the inventory uses the group convention described above there are a few special cases: