Update CI files

Author: pulpbot

.ci/ansible/Containerfile.j2

@@ -1,34 +1,28 @@
-FROM {{ ci_base | default(pulp_default_container) }}
+FROM {{ image.ci_base }}
+{%- if image.webserver_snippet %}
 
-# Add source directories to container
-{% for item in plugins %}
-ADD ./{{ item.name }} ./{{ item.name }}
-{% endfor %}
+ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf
+{%- endif %}
+
+{%- for item in extra_files | default([]) %}
 
-{% for item in extra_files | default([]) %}
 ADD ./{{ item.origin }} {{ item.destination }}
-{% endfor %}
+{%- endfor %}
 
 # This MUST be the ONLY call to pip install in inside the container.
 RUN pip3 install --upgrade pip setuptools wheel && \
   rm -rf /root/.cache/pip && \
-  pip3 install
-{%- if s3_test | default(false) -%}
-{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue
+  pip3 install {{ image.source }}
+{%- if image.upperbounds | default(false) -%}
+{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt
 {%- endif -%}
-{%- for item in plugins -%}
-{{ " " }}{{ item.source }}
-{%- if item.upperbounds | default(false) -%}
-{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt
+{%- if image.lowerbounds | default(false) -%}
+{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt
 {%- endif -%}
-{%- if item.lowerbounds | default(false) -%}
-{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt
+{%- if image.ci_requirements | default(false) -%}
+{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt
 {%- endif -%}
-{%- if item.ci_requirements | default(false) -%}
-{{ " " }}-r ./{{ item.name }}/ci_requirements.txt
-{%- endif -%}
-{%- endfor %}
-{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \
+{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt && \
   rm -rf /root/.cache/pip
 
 {% if pulp_env is defined and pulp_env %}
@@ -46,11 +40,12 @@ ENV {{ key | upper }}={{ value }}
 USER pulp:pulp
 RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \
        /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link
+
+RUN mkdir /var/lib/pulp/.config
 USER root:root
 
-{% for item in plugins %}
-RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \
-    ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true
-{% endfor %}
+# Lots of plugins try to use this path, and throw warnings if they cannot access it.
+RUN mkdir /.pytest_cache
+RUN chown pulp:pulp /.pytest_cache
 
 ENTRYPOINT ["/init"]

.ci/ansible/build_container.yaml

@@ -1,14 +1,15 @@
 # Ansible playbook to create the pulp service containers image
 ---
-- hosts: localhost
+- name: "Build CI Container Image"
+  hosts: "localhost"
   gather_facts: false
   vars_files:
-    - vars/main.yaml
+    - "vars/main.yaml"
   tasks:
     - name: "Generate Containerfile from template"
-      template:
-        src: Containerfile.j2
-        dest: Containerfile
+      ansible.builtin.template:
+        src: "Containerfile.j2"
+        dest: "Containerfile"
 
     - name: "Build pulp image"
       # We build from the ../.. (parent dir of pulpcore git repo) Docker build
@@ -19,9 +20,20 @@
       # 1-off-builds and CI purposes (which has no cache across CI runs.)
       # Run build.yaml with -e cache=false if your builds are using outdated
       # layers.
-      command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}/Containerfile ../../.."
+      ansible.builtin.command:
+        argv:
+          - "docker"
+          - "build"
+          - "--network"
+          - "host"
+          - "--no-cache={{ not cache | default(true) | bool }}"
+          - "-t"
+          - "{{ image.name }}:{{ image.tag }}"
+          - "-f"
+          - "{{ playbook_dir }}/Containerfile"
+          - "../../.."
 
     - name: "Clean image cache"
-      docker_prune:
-        images : true
+      community.docker.docker_prune:
+        images: true
 ...

.ci/ansible/settings.py.j2

@@ -10,10 +10,6 @@ REDIS_HOST = "localhost"
 REDIS_PORT = 6379
 ANALYTICS = False
 
-{% if api_root is defined %}
-API_ROOT = {{ api_root | repr }}
-{% endif %}
-
 {% if pulp_settings %}
 {% for key, value in pulp_settings.items() %}
 {{ key | upper }} = {{ value | repr }}

.ci/ansible/start_container.yaml

@@ -1,109 +1,127 @@
 # Ansible playbook to start the pulp service container and its supporting services
 ---
-- hosts: localhost
+- name: "Start CI Containers"
+  hosts: "localhost"
   gather_facts: false
   vars_files:
-    - vars/main.yaml
+    - "vars/main.yaml"
   tasks:
     - name: "Create Settings Directories"
-      file:
+      ansible.builtin.file:
         path: "{{ item }}"
-        state: directory
+        state: "directory"
         mode: "0755"
       loop:
-        - settings
-        - ssh
-        - ~/.config/pulp_smash
+        - "settings"
 
     - name: "Generate Pulp Settings"
-      template:
-        src: settings.py.j2
-        dest: settings/settings.py
-
-    - name: "Configure pulp-smash"
-      copy:
-        src: smash-config.json
-        dest: ~/.config/pulp_smash/settings.json
+      ansible.builtin.template:
+        src: "settings.py.j2"
+        dest: "settings/settings.py"
 
     - name: "Setup docker networking"
-      docker_network:
-        name: pulp_ci_bridge
+      community.docker.docker_network:
+        name: "pulp_ci_bridge"
 
     - name: "Start Service Containers"
-      docker_container:
+      community.docker.docker_container:
         name: "{{ item.name }}"
         image: "{{ item.image }}"
         auto_remove: true
         recreate: true
         privileged: true
         networks:
-          - name: pulp_ci_bridge
+          - name: "pulp_ci_bridge"
             aliases: "{{ item.name }}"
         volumes: "{{ item.volumes | default(omit) }}"
         env: "{{ item.env | default(omit) }}"
         command: "{{ item.command | default(omit) }}"
-        state: started
+        state: "started"
       loop: "{{ services | default([]) }}"
 
     - name: "Retrieve Docker Network Info"
-      docker_network_info:
-        name: pulp_ci_bridge
-      register: pulp_ci_bridge_info
+      community.docker.docker_network_info:
+        name: "pulp_ci_bridge"
+      register: "pulp_ci_bridge_info"
 
     - name: "Update /etc/hosts"
-      lineinfile:
-        path: /etc/hosts
+      ansible.builtin.lineinfile:
+        path: "/etc/hosts"
         regexp: "\\s{{ item.value.Name }}\\s*$"
-        line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}"
+        line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}"
       loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}"
       become: true
 
     - name: "Create Pulp Bucket"
       amazon.aws.s3_bucket:
         aws_access_key: "{{ minio_access_key }}"
         aws_secret_key: "{{ minio_secret_key }}"
-        s3_url: "http://minio:9000"
-        region: eu-central-1
-        name: pulp3
-        state: present
-      when: s3_test | default(false)
+        endpoint_url: "http://minio:9000"
+        region: "eu-central-1"
+        name: "pulp3"
+        state: "present"
+      when: "s3_test | default(false)"
+
+    - name: "Wait on Services"
+      block:
+        - name: "Wait for azurite"
+          ansible.builtin.uri:
+            url: "http://ci-azurite:10000/"
+            status_code:
+              - 200
+              - 400
+          when: "azure_test | default(false)"
+          retries: 2
+          delay: 5
 
-    - block:
         - name: "Wait for Pulp"
-          uri:
-            url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/"
-            follow_redirects: all
-            validate_certs: no
-          register: result
-          until: result.status == 200
+          ansible.builtin.uri:
+            url: "http://pulp{{ pulp_scenario_settings.api_root | default(pulp_settings.api_root | default('\/pulp\/', True), True) }}api/v3/status/"
+            follow_redirects: "all"
+            validate_certs: "no"
+          register: "result"
+          until: "result.status == 200"
           retries: 12
           delay: 5
       rescue:
         - name: "Output pulp container log"
-          command: "docker logs pulp"
+          ansible.builtin.command:
+            cmd: "docker logs pulp"
           failed_when: true
 
     - name: "Check version of component being tested"
-      assert:
+      ansible.builtin.assert:
         that:
-          - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver)
+          - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)"
         fail_msg: |
           Component {{ item.app_label }} was expected to be installed in version {{ component_version }}.
           Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}.
       loop: "{{ 'plugins' | ansible.builtin.extract(lookup('ansible.builtin.file', '../../template_config.yml') | from_yaml) }}"
 
     - name: "Set pulp password in .netrc"
-      copy:
+      ansible.builtin.copy:
         dest: "~/.netrc"
         content: |
           machine pulp
           login admin
           password password
 
-- hosts: pulp
+- name: "Prepare Pulp Application Container"
+  hosts: "pulp"
   gather_facts: false
   tasks:
+    - name: "Create directory for pulp-smash config"
+      ansible.builtin.file:
+        path: "/var/lib/pulp/.config/pulp_smash/"
+        state: "directory"
+        mode: "0755"
+
+    - name: "Configure pulp-smash"
+      ansible.builtin.copy:
+        src: "smash-config.json"
+        dest: "/var/lib/pulp/.config/pulp_smash/settings.json"
+
     - name: "Set pulp admin password"
-      command:
+      ansible.builtin.command:
         cmd: "pulpcore-manager reset-admin-password --password password"
 ...

.ci/assets/ci_constraints.txt

@@ -1,5 +1,6 @@
 # Pulpcore versions without the openapi command do no longer work in the CI
-pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*
+# Pulpcore versions without the django 5 storage compatibility will fail, >3.63,<3.70
+pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*,!=3.64.*,!=3.65.*,!=3.66.*,!=3.67.*,!=3.68.*,!=3.69.*
 
 
 tablib!=3.6.0
@@ -8,3 +9,11 @@ tablib!=3.6.0
 
 multidict!=6.3.0
 # This release failed the lower bounds test for some case sensitivity in CIMultiDict.
+
+
+azure-storage-blob!=12.28.*
+# Apparently does not work with current azurite.
+
+
+pycares<5
+# older aiodns versions don't pin pycares UB, and are broken by pycares>=5