Update CI files

Author: pulpbot

.ci/ansible/Containerfile.j2

@@ -1,34 +1,28 @@
-FROM {{ ci_base | default(pulp_default_container) }}
+FROM {{ image.ci_base }}
+{%- if image.webserver_snippet %}
 
-# Add source directories to container
-{% for item in plugins %}
-ADD ./{{ item.name }} ./{{ item.name }}
-{% endfor %}
+ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf
+{%- endif %}
+
+{%- for item in extra_files | default([]) %}
 
-{% for item in extra_files | default([]) %}
 ADD ./{{ item.origin }} {{ item.destination }}
-{% endfor %}
+{%- endfor %}
 
 # This MUST be the ONLY call to pip install in inside the container.
 RUN pip3 install --upgrade pip setuptools wheel && \
   rm -rf /root/.cache/pip && \
-  pip3 install
-{%- if s3_test | default(false) -%}
-{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue
-{%- endif -%}
-{%- for item in plugins -%}
-{{ " " }}{{ item.source }}
-{%- if item.upperbounds | default(false) -%}
-{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt
+  pip3 install {{ image.source }}
+{%- if image.upperbounds | default(false) -%}
+{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt
 {%- endif -%}
-{%- if item.lowerbounds | default(false) -%}
-{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt
+{%- if image.lowerbounds | default(false) -%}
+{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt
 {%- endif -%}
-{%- if item.ci_requirements | default(false) -%}
-{{ " " }}-r ./{{ item.name }}/ci_requirements.txt
+{%- if image.ci_requirements | default(false) -%}
+{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt
 {%- endif -%}
-{%- endfor %}
-{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \
+{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt && \
   rm -rf /root/.cache/pip
 
 {% if pulp_env is defined and pulp_env %}
@@ -46,11 +40,8 @@ ENV {{ key | upper }}={{ value }}
 USER pulp:pulp
 RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \
        /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link
-USER root:root
 
-{% for item in plugins %}
-RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \
-    ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true
-{% endfor %}
+RUN mkdir /var/lib/pulp/.config
+USER root:root
 
 ENTRYPOINT ["/init"]

.ci/ansible/build_container.yaml

@@ -1,15 +1,14 @@
 # Ansible playbook to create the pulp service containers image
 ---
-- hosts: localhost
+- hosts: "localhost"
   gather_facts: false
   vars_files:
-    - vars/main.yaml
+    - "vars/main.yaml"
   tasks:
     - name: "Generate Containerfile from template"
-      template:
-        src: Containerfile.j2
-        dest: Containerfile
-
+      ansible.builtin.template:
+        src: "Containerfile.j2"
+        dest: "Containerfile"
     - name: "Build pulp image"
       # We build from the ../.. (parent dir of pulpcore git repo) Docker build
       # "context" so that repos like pulp-smash are accessible to Docker

.ci/ansible/start_container.yaml

@@ -1,33 +1,26 @@
 # Ansible playbook to start the pulp service container and its supporting services
 ---
-- hosts: localhost
+- hosts: "localhost"
   gather_facts: false
   vars_files:
-    - vars/main.yaml
+    - "vars/main.yaml"
   tasks:
     - name: "Create Settings Directories"
-      file:
+      ansible.builtin.file:
         path: "{{ item }}"
-        state: directory
+        state: "directory"
         mode: "0755"
       loop:
-        - settings
-        - ssh
-        - ~/.config/pulp_smash
+        - "settings"
 
     - name: "Generate Pulp Settings"
       template:
-        src: settings.py.j2
-        dest: settings/settings.py
-
-    - name: "Configure pulp-smash"
-      copy:
-        src: smash-config.json
-        dest: ~/.config/pulp_smash/settings.json
+        src: "settings.py.j2"
+        dest: "settings/settings.py"
 
     - name: "Setup docker networking"
       docker_network:
-        name: pulp_ci_bridge
+        name: "pulp_ci_bridge"
 
     - name: "Start Service Containers"
       docker_container:
@@ -37,24 +30,24 @@
         recreate: true
         privileged: true
         networks:
-          - name: pulp_ci_bridge
+          - name: "pulp_ci_bridge"
             aliases: "{{ item.name }}"
         volumes: "{{ item.volumes | default(omit) }}"
         env: "{{ item.env | default(omit) }}"
         command: "{{ item.command | default(omit) }}"
-        state: started
+        state: "started"
       loop: "{{ services | default([]) }}"
 
     - name: "Retrieve Docker Network Info"
       docker_network_info:
-        name: pulp_ci_bridge
-      register: pulp_ci_bridge_info
+        name: "pulp_ci_bridge"
+      register: "pulp_ci_bridge_info"
 
     - name: "Update /etc/hosts"
       lineinfile:
-        path: /etc/hosts
+        path: "/etc/hosts"
         regexp: "\\s{{ item.value.Name }}\\s*$"
-        line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}"
+        line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}"
       loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}"
       become: true
 
@@ -63,19 +56,19 @@
         aws_access_key: "{{ minio_access_key }}"
         aws_secret_key: "{{ minio_secret_key }}"
         s3_url: "http://minio:9000"
-        region: eu-central-1
-        name: pulp3
-        state: present
-      when: s3_test | default(false)
+        region: "eu-central-1"
+        name: "pulp3"
+        state: "present"
+      when: "s3_test | default(false)"
 
     - block:
         - name: "Wait for Pulp"
           uri:
             url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/"
-            follow_redirects: all
-            validate_certs: no
-          register: result
-          until: result.status == 200
+            follow_redirects: "all"
+            validate_certs: "no"
+          register: "result"
+          until: "result.status == 200"
           retries: 12
           delay: 5
       rescue:
@@ -86,7 +79,7 @@
     - name: "Check version of component being tested"
       assert:
         that:
-          - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver)
+          - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)"
         fail_msg: |
           Component {{ item.app_label }} was expected to be installed in version {{ component_version }}.
           Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}.
@@ -100,9 +93,20 @@
           login admin
           password password
 
-- hosts: pulp
+- hosts: "pulp"
   gather_facts: false
   tasks:
+    - name: "Create directory for pulp-smash config"
+      ansible.builtin.file:
+        path: "/var/lib/pulp/.config/pulp_smash/"
+        state: "directory"
+        mode: "0755"
+
+    - name: "Configure pulp-smash"
+      ansible.builtin.copy:
+        src: "smash-config.json"
+        dest: "/var/lib/pulp/.config/pulp_smash/settings.json"
+
     - name: "Set pulp admin password"
       command:
         cmd: "pulpcore-manager reset-admin-password --password password"

.ci/assets/ci_constraints.txt

@@ -1,5 +1,6 @@
 # Pulpcore versions without the openapi command do no longer work in the CI
-pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*
+# Pulpcore versions without the django 5 storage compatibility will fail, >3.63,<3.70
+pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*,!=3.64.*,!=3.65.*,!=3.66.*,!=3.67.*,!=3.68.*,!=3.69.*
 
 
 tablib!=3.6.0
@@ -8,3 +9,11 @@ tablib!=3.6.0
 
 multidict!=6.3.0
 # This release failed the lower bounds test for some case sensitivity in CIMultiDict.
+
+
+azure-storage-blob!=12.28.*
+# Apparently does not work with current azurite.
+
+
+pycares<5
+# older aiodns versions don't pin pycares UB, and are broken by pycares>=5

.ci/scripts/check_release.py

@@ -1,11 +1,22 @@
 #!/usr/bin/env python
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#     "gitpython>=3.1.46,<3.2.0",
+#     "packaging>=26.0,<26.1",
+#     "pyyaml>=6.0.3,<6.1.0",
+# ]
+# ///
 
 import argparse
 import re
 import os
+import sys
 import tomllib
-import yaml
+import typing as t
 from pathlib import Path
+
+import yaml
 from packaging.version import Version
 from git import Repo
 
@@ -14,7 +25,7 @@
 Z_CHANGELOG_EXTS = [".bugfix", ".misc"]
 
 
-def options():
+def options() -> argparse.Namespace:
     """Check which branches need a release."""
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -33,13 +44,13 @@ def options():
     return parser.parse_args()
 
 
-def template_config():
+def template_config() -> dict[str, t.Any]:
     # Assume this script lies in .ci/scripts
     path = Path(__file__).absolute().parent.parent.parent / "template_config.yml"
     return yaml.safe_load(path.read_text())
 
 
-def current_version(repo, commitish):
+def current_version(repo: Repo, commitish: str) -> Version:
     try:
         pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml"))
         try:
@@ -53,7 +64,7 @@ def current_version(repo, commitish):
     return Version(current_version)
 
 
-def check_pyproject_dependencies(repo, from_commit, to_commit):
+def check_pyproject_dependencies(repo: Repo, from_commit: str, to_commit: str) -> list[str]:
     try:
         new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml"))
         try:
@@ -74,8 +85,8 @@ def check_pyproject_dependencies(repo, from_commit, to_commit):
         return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."]
 
 
-def main(options, template_config):
-    DEFAULT_BRANCH = template_config["plugin_default_branch"]
+def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int:
+    DEFAULT_BRANCH: str = template_config["plugin_default_branch"]
 
     repo = Repo()
 
@@ -88,9 +99,9 @@ def main(options, template_config):
 
     # Warning: This will not work if branch names contain "/" but we don't really care here.
     heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")]
-    available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)]
-    available_branches.sort(key=lambda ver: Version(ver))
-    available_branches.append(DEFAULT_BRANCH)
+    available_branches = sorted(
+        {h for h in heads if re.fullmatch(RELEASE_BRANCH_REGEX, h)}, key=lambda ver: Version(ver)
+    ) + [DEFAULT_BRANCH]
 
     branches = options.branches
     if branches == "supported":
@@ -105,7 +116,10 @@ def main(options, template_config):
 
     if diff := branches - set(available_branches):
         print(f"Supplied branches contains non-existent branches! {diff}")
-        exit(1)
+        return 1
+
+    branches = [branch for branch in available_branches if branch in branches]
+    branches.reverse()
 
     print(f"Checking for releases on branches: {branches}")
 
@@ -170,6 +184,8 @@ def main(options, template_config):
     if len(releases) == 0:
         print("No new releases to perform.")
 
+    return 0
+
 
 if __name__ == "__main__":
-    main(options(), template_config())
+    sys.exit(main(options(), template_config()))

.ci/scripts/check_requirements.py

@@ -9,7 +9,6 @@
 import warnings
 from packaging.requirements import Requirement
 
-
 CHECK_MATRIX = [
     ("pyproject.toml", True, True, True),
     ("requirements.txt", True, True, True),

.ci/scripts/clean_gh_release_notes.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# This script is running with elevated privileges from the main branch against pull requests.
+#
+# It cleans the input from artifacts which are used by the pulp documentation internally,
+# but clutter for GitHub releases
+
+import sys
+
+NOTE = """
+> [!NOTE]
+> Official changes are available on [Pulp docs]({docs_url})\
+"""
+
+
+def main():
+    plugin_name = sys.argv[1]
+    version_str = sys.argv[2]
+    docs_url = f"https://pulpproject.org/{plugin_name}/changes/#{version_str}"
+    note_added = False
+    for line in sys.stdin:
+        if line.endswith("\n"):
+            line = line[:-1]
+        if line.startswith("#"):
+            print(line.split(" {: #")[0])
+            if not note_added and version_str in line:
+                print(NOTE.format(docs_url=docs_url))
+                note_added = True
+        else:
+            print(line)
+
+
+if __name__ == "__main__":
+    main()