Update CI settings to partially use new storage settings

Author: pedro-psb

.ci/ansible/settings.py.j2

@@ -27,19 +27,30 @@ API_ROOT = {{ api_root | repr }}
 {% endif %}
 
 {% if s3_test | default(false) %}
-DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
-MEDIA_ROOT = ""
-AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
-AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
-AWS_S3_REGION_NAME = "eu-central-1"
-AWS_S3_ADDRESSING_STYLE = "path"
+MEDIA_ROOT: ""
 S3_USE_SIGV4 = True
-AWS_S3_SIGNATURE_VERSION = "s3v4"
-AWS_STORAGE_BUCKET_NAME = "pulp3"
-AWS_S3_ENDPOINT_URL = "http://minio:9000"
-AWS_DEFAULT_ACL = "@none None"
+STORAGES = {
+    "default": {
+        "BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
+        "OPTIONS": {
+            "access_key": "{{ minio_access_key }}",
+            "secret_key": "{{ minio_secret_key }}",
+            "region_name": "eu-central-1",
+            "addressing_style": "path",
+            "signature_version": "s3v4",
+            "bucket_name": "pulp3",
+            "endpoint_url": "http://minio:9000",
+            "default_acl": "@none None",
+        },
+    },
+    "staticfiles": {
+      "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+    },
+}
 {% endif %}
 
+# This is using DEFAULT_FILE_STORAGE to test both usages.
+# Remove when DEFAULT_FILE_STORAGE is completely removed.
 {% if azure_test | default(false) %}
 DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
 MEDIA_ROOT = ""

pulpcore/app/serializers/domain.py

@@ -42,7 +42,10 @@ def to_representation(self, instance):
         # Should I convert back the saved settings to their Setting names for to_representation?
         if getattr(self.context.get("domain", None), "name", None) == "default":
             for setting_name, field in self.SETTING_MAPPING.items():
-                if value := getattr(settings, setting_name.upper(), None):
+                value = getattr(settings, setting_name, None) or settings.STORAGES["default"].get(
+                    "OPTIONS", {}
+                ).get(field)
+                if value:
                     instance[field] = value
         return super().to_representation(instance)
 

pulpcore/pytest_plugin.py

@@ -603,13 +603,13 @@ def _settings_factory(storage_class=None, storage_settings=None):
         keys = dict()
         keys["pulpcore.app.models.storage.FileSystem"] = ["MEDIA_ROOT", "MEDIA_URL"]
         keys["storages.backends.s3boto3.S3Boto3Storage"] = [
-            "AWS_ACCESS_KEY_ID",
-            "AWS_SECRET_ACCESS_KEY",
-            "AWS_S3_ENDPOINT_URL",
-            "AWS_S3_ADDRESSING_STYLE",
-            "AWS_S3_SIGNATURE_VERSION",
-            "AWS_S3_REGION_NAME",
-            "AWS_STORAGE_BUCKET_NAME",
+            "access_key",
+            "secret_key",
+            "endpoint_url",
+            "addressing_style",
+            "signature_version",
+            "region_name",
+            "bucket_name",
         ]
         keys["storages.backends.azure_storage.AzureStorage"] = [
             "AZURE_ACCOUNT_NAME",
@@ -622,8 +622,16 @@ def _settings_factory(storage_class=None, storage_settings=None):
         ]
         settings = storage_settings or dict()
         backend = storage_class or pulp_settings.STORAGES["default"]["BACKEND"]
-        for key in keys[backend]:
-            if key not in settings:
+        not_defined_settings = (k for k in keys[backend] if k not in settings)
+        # The CI configures s3 with STORAGES and Azure with legacy
+        # Move all to STORAGES structure on DEFAULT_FILE_STORAGE removal
+        if backend == "storages.backends.s3boto3.S3Boto3Storage":
+            storages_dict = getattr(pulp_settings, "STORAGES", {})
+            storage_options = storages_dict.get("default", {}).get("OPTIONS", {})
+            for key in not_defined_settings:
+                settings[key] = storage_options.get(key)
+        else:
+            for key in not_defined_settings:
                 settings[key] = getattr(pulp_settings, key, None)
         return backend, settings