Bump pulpcore upperbound to <3.100

Author: gerrod3

.ci/ansible/settings.py.j2

@@ -26,10 +26,22 @@ API_ROOT = {{ api_root | repr }}
 {% endfor %}
 {% endif %}
 
-{% if s3_test | default(false) %}
-MEDIA_ROOT: ""
-S3_USE_SIGV4 = True
-{% if test_storages_compat_layer is defined and test_storages_compat_layer %}
+{# =======================================
+Macros for legacy and new storage settings
+========================================== -#}
+
+{%- macro s3_settings(legacy) -%}
+    {%- if legacy %}
+DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
+AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
+AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
+AWS_S3_REGION_NAME = "eu-central-1"
+AWS_S3_ADDRESSING_STYLE = "path"
+AWS_S3_SIGNATURE_VERSION = "s3v4"
+AWS_STORAGE_BUCKET_NAME = "pulp3"
+AWS_S3_ENDPOINT_URL = "http://minio:9000"
+AWS_DEFAULT_ACL = "@none None"
+    {%- else %}
 STORAGES = {
     "default": {
         "BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
@@ -48,34 +60,93 @@ STORAGES = {
       "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
     },
 }
-{% else %}
-DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
-AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
-AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
-AWS_S3_REGION_NAME = "eu-central-1"
-AWS_S3_ADDRESSING_STYLE = "path"
-AWS_S3_SIGNATURE_VERSION = "s3v4"
-AWS_STORAGE_BUCKET_NAME = "pulp3"
-AWS_S3_ENDPOINT_URL = "http://minio:9000"
-AWS_DEFAULT_ACL = "@none None"
-{% endif %}
-{% endif %}
+    {%- endif %}
+{%- endmacro -%}
 
-{% if azure_test | default(false) %}
+{%- macro azure_settings(legacy) -%}
+    {%- if legacy %}
 DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
-MEDIA_ROOT = ""
 AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
 AZURE_ACCOUNT_NAME = "devstoreaccount1"
 AZURE_CONTAINER =  "pulp-test"
 AZURE_LOCATION = "pulp3"
 AZURE_OVERWRITE_FILES = True
 AZURE_URL_EXPIRATION_SECS = 120
 AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;'
-{% endif %}
+    {%- else %}
+STORAGES = {
+    "default": {
+        "BACKEND": "storages.backends.azure_storage.AzureStorage",
+        "OPTIONS": {
+            "account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
+            "account_name": "devstoreaccount1",
+            "location": "pulp3",
+            "azure_container": "pulp-test",
+            "overwrite_files": True,
+            "expiration_secs": 120,
+            "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;",
+        },
+    },
+    "staticfiles": {
+      "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+    },
+}
+    {%- endif %}
+{%- endmacro -%}
 
-{% if gcp_test | default(false) %}
+{%- macro gcp_settings(legacy) -%}
+    {%- if legacy %}
 DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
-MEDIA_ROOT = ""
 GS_BUCKET_NAME = "gcppulp"
 GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443"
+    {%- else %}
+STORAGES = {
+    "default": {
+        "BACKEND": "storages.backends.gcloud.GoogleCloudStorage",
+        "OPTIONS": {
+            "bucket_name": "gcppulp",
+            "custom_endpoint": "http://ci-gcp:4443",
+        },
+    },
+    "staticfiles": {
+      "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+    },
+}
+    {%- endif %}
+{%- endmacro -%}
+
+{#- ==========================================
+Render according to test_storages_compat_layer
+==============================================
+
+Case 1) test_storages_compat_layer is unset
+- All storages render the legacy setting
+- Branches using pulpcore <3.70 must leave this key unset (use legacy)
+
+Case 2) test_storages_compat_layer is True
+- To tests both work, only one setting uses the new storage setting
+- Branches using pulpcore >=3.70,<3.85 must set this key to True (test both)
+
+Case 3) test_storages_compat_layer is False
+- All storages render the new setting
+- Branches using pulpcore >=3.85 must set this key to False (use new)
+-#}
+
+{% if s3_test | default(false) or azure_test | default(false) or gcp_test | default(false)%}
+MEDIA_ROOT=""
 {% endif %}
+{%- if test_storages_compat_layer is not defined -%}
+    {%- if s3_test | default(false) -%}{{ s3_settings(legacy=True) }}{%- endif -%}
+    {%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%}
+    {%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%}
+{%- else -%}
+    {%- if test_storages_compat_layer is true -%}
+        {%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%}
+        {%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%}
+        {%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%}
+    {%- elif test_storages_compat_layer is false -%}
+        {%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%}
+        {%- if azure_test | default(false) -%}{{ azure_settings(legacy=False) }}{%- endif -%}
+        {%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=False) }}{%- endif -%}
+    {%- endif -%}
+{%- endif -%}

.github/workflows/scripts/install.sh

@@ -97,7 +97,7 @@ minio_access_key: "'$MINIO_ACCESS_KEY'"\
 minio_secret_key: "'$MINIO_SECRET_KEY'"\
 pulp_scenario_settings: {"domain_enabled": true, "flatpak_index": false, "token_auth_disabled": true}\
 pulp_scenario_env: {}\
-test_storages_compat_layer: true\
+test_storages_compat_layer: false\
 ' vars/main.yaml
   export PULP_API_ROOT="/rerouted/djnd/"
 fi
@@ -112,6 +112,7 @@ if [ "$TEST" = "azure" ]; then
   sed -i -e '$a azure_test: true\
 pulp_scenario_settings: {"content_origin": null, "domain_enabled": true, "flatpak_index": true}\
 pulp_scenario_env: {}\
+test_storages_compat_layer: false\
 ' vars/main.yaml
 fi
 

CHANGES/+3.85-compat.feature

@@ -0,0 +1 @@
+Bump pulpcore upperbound to <3.100. pulp_container is now a Python >=3.11 project.

pulp_container/app/registry_api.py

@@ -28,7 +28,7 @@
 
 from pulpcore.plugin.models import Artifact, ContentArtifact, UploadChunk
 from pulpcore.plugin.files import PulpTemporaryUploadedFile
-from pulpcore.plugin.tasking import add_and_remove, dispatch
+from pulpcore.plugin.tasking import dispatch
 from pulpcore.plugin.util import get_objects_for_user, get_url, get_domain
 from pulpcore.plugin.exceptions import TimeoutException
 
@@ -74,6 +74,7 @@
     S3StorageRedirects,
     AzureStorageRedirects,
 )
+from pulp_container.app.tasks import aadd_and_remove
 from pulp_container.app.token_verification import (
     RegistryAuthentication,
     TokenAuthentication,
@@ -1137,7 +1138,7 @@ def handle_safe_method(self, request, path, pk):
                     add_content_units = self.get_content_units_to_add(manifest, tag)
 
                     dispatch(
-                        add_and_remove,
+                        aadd_and_remove,
                         exclusive_resources=[repository],
                         kwargs={
                             "repository_pk": str(repository.pk),
@@ -1393,7 +1394,7 @@ def put(self, request, path, pk=None):
             remove_content_units = [str(pk) for pk in tags_to_remove.values_list("pk")]
 
             immediate_task = dispatch(
-                add_and_remove,
+                aadd_and_remove,
                 exclusive_resources=[repository],
                 kwargs={
                     "repository_pk": str(repository.pk),
@@ -1563,7 +1564,7 @@ def put(self, request, path, pk):
             signature.touch()
 
         immediate_task = dispatch(
-            add_and_remove,
+            aadd_and_remove,
             exclusive_resources=[repository],
             kwargs={
                 "repository_pk": str(repository.pk),

pulp_container/app/tasks/__init__.py

@@ -1,4 +1,4 @@
-from .download_image_data import download_image_data  # noqa
+from .download_image_data import aadd_and_remove, download_image_data  # noqa
 from .builder import build_image_from_containerfile, build_image  # noqa
 from .recursive_add import recursive_add_content  # noqa
 from .recursive_remove import recursive_remove_content  # noqa

pulp_container/app/tasks/download_image_data.py

@@ -1,7 +1,10 @@
 import json
 import logging
 
+from asgiref.sync import sync_to_async
+
 from pulpcore.plugin.stages import DeclarativeContent
+from pulpcore.plugin.tasking import add_and_remove
 
 from pulp_container.app.models import ContainerRemote, ContainerRepository, Tag
 from pulp_container.app.utils import determine_media_type_from_json
@@ -13,6 +16,10 @@
 log = logging.getLogger(__name__)
 
 
+async def aadd_and_remove(*args, **kwargs):
+    return await sync_to_async(add_and_remove)(*args, **kwargs)
+
+
 def download_image_data(repository_pk, remote_pk, raw_text_manifest_data, tag_name):
     repository = ContainerRepository.objects.get(pk=repository_pk)
     remote = ContainerRemote.objects.get(pk=remote_pk)

pyproject.toml

@@ -20,12 +20,11 @@ classifiers=[
   "Framework :: Django",
   "Programming Language :: Python",
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.9",
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.11"
 dependencies = [
   "jsonschema>=4.4,<4.25",
-  "pulpcore>=3.73.2,<3.85",
+  "pulpcore>=3.73.2,<3.100",
   "pyjwt[crypto]>=2.4,<2.11",
 ]
 

template_config.yml

@@ -84,6 +84,6 @@ test_lowerbounds: true
 test_performance: false
 test_reroute: true
 test_s3: true
-test_storages_compat_layer: true
+test_storages_compat_layer: false
 use_issue_template: true