Add support for rename of S3 storage backend

fixes: #7228
(cherry picked from commit 0d632d4)

Author: gerrod3

CHANGES/7228.bugfix

@@ -0,0 +1 @@
+Fixed not supporting the new rename of the S3 storage backend.

docs/admin/guides/configure-pulp/configure-storages.md

@@ -103,7 +103,7 @@ When creating a `Domain` you can use the following payload:
 ```json
 {
   "name": "cloudfront_storage",
-  "storage_class": "storages.backends.s3boto3.S3Boto3Storage",
+  "storage_class": "storages.backends.s3.S3Storage",
   "storage_settings": {
     "access_key": "{aws_access_key}",
     "secret_key": "{aws_secret_key}",
@@ -124,14 +124,14 @@ When creating a `Domain` you can use the following payload:
 # We need to format our key to a JSON friendly format
 export CLOUDFRONT_KEY=$(cat keyfile.pem | jq -sR .)
 
-curl -X POST $BASE_HREF/pulp/default/api/v3/domains/ -d '{"name": "cloudfront_storage", "storage_class": "storages.backends.s3boto3.S3Boto3Storage", "storage_settings": {"access_key": "{aws_access_key}", "secret_key": "{aws_secret_key}", "bucket_name": "{aws_bucket_name}", "region_name": "{aws_region_name}", "default_acl": "private", "cloudfront_key_id": "{aws_cloudfront_key_id}", "cloudfront_key": '"$CLOUDFRONT_KEY"', "custom_domain": "{aws_cloudfront_custom_domain}"}, "redirect_to_object_storage": true, "hide_guarded_distributions": false}' -H 'Content-Type: application/json'
+curl -X POST $BASE_HREF/pulp/default/api/v3/domains/ -d '{"name": "cloudfront_storage", "storage_class": "storages.backends.s3.S3Storage", "storage_settings": {"access_key": "{aws_access_key}", "secret_key": "{aws_secret_key}", "bucket_name": "{aws_bucket_name}", "region_name": "{aws_region_name}", "default_acl": "private", "cloudfront_key_id": "{aws_cloudfront_key_id}", "cloudfront_key": '"$CLOUDFRONT_KEY"', "custom_domain": "{aws_cloudfront_custom_domain}"}, "redirect_to_object_storage": true, "hide_guarded_distributions": false}' -H 'Content-Type: application/json'
 ```
 
 === Create a new Domain using pulp-cli
 ```bash
 export CLOUDFRONT_KEY=$(cat keyfile.pem | jq -sR .)
 
-pulp domain create --name cloudfront_test --storage-class storages.backends.s3boto3.S3Boto3Storage --storage-settings '{"access_key": "{aws_access_key}", "secret_key": "{aws_secret_key}", "bucket_name": "{aws_bucket_name}", "region_name": "{aws_region_name}", "default_acl": "private", "cloudfront_key_id": "{aws_cloudfront_key_id}", "cloudfront_key": '"$CLOUDFRONT_KEY"', "custom_domain": "{aws_cloudfront_custom_domain}"}'
+pulp domain create --name cloudfront_test --storage-class storages.backends.s3.S3Storage --storage-settings '{"access_key": "{aws_access_key}", "secret_key": "{aws_secret_key}", "bucket_name": "{aws_bucket_name}", "region_name": "{aws_region_name}", "default_acl": "private", "cloudfront_key_id": "{aws_cloudfront_key_id}", "cloudfront_key": '"$CLOUDFRONT_KEY"', "custom_domain": "{aws_cloudfront_custom_domain}"}'
 ```
 
 Create your remotes, repositories, and distributions under this domain and the requests will be redirected to the

pulpcore/app/serializers/domain.py

@@ -22,7 +22,8 @@
 BACKEND_CHOICES = (
     ("pulpcore.app.models.storage.FileSystem", "Use local filesystem as storage"),
     # ("pulpcore.app.models.storage.PulpSFTPStorage", "Use SFTP server as storage"),
-    ("storages.backends.s3boto3.S3Boto3Storage", "Use Amazon S3 as storage"),
+    ("storages.backends.s3boto3.S3Boto3Storage", "Use Amazon S3 as storage [deprecated]"),
+    ("storages.backends.s3.S3Storage", "Use Amazon S3 as storage"),
     ("storages.backends.azure_storage.AzureStorage", "Use Azure Blob as storage"),
     # ("storages.backends.gcloud.GoogleCloudStorage", "Use Google Cloud as storage"),
 )
@@ -159,7 +160,7 @@ def create(self, validated_data):
 class AmazonS3SettingsSerializer(BaseSettingsClass):
     """A Serializer for Amazon S3 storage settings."""
 
-    STORAGE_CLASS = "storages.backends.s3boto3.S3Boto3Storage"
+    STORAGE_CLASS = "storages.backends.s3.S3Storage"
     SETTING_MAPPING = {
         "aws_s3_access_key_id": "access_key",
         "aws_access_key_id": "access_key",

pulpcore/app/util.py

@@ -516,7 +516,10 @@ def get_artifact_url(artifact, headers=None, http_method=None):
         or not artifact_domain.redirect_to_object_storage
     ):
         return _artifact_serving_distribution().artifact_url(artifact)
-    elif artifact_domain.storage_class == "storages.backends.s3boto3.S3Boto3Storage":
+    elif artifact_domain.storage_class in (
+        "storages.backends.s3boto3.S3Boto3Storage",
+        "storages.backends.s3.S3Storage",
+    ):
         parameters = {"ResponseContentDisposition": content_disposition}
         if headers and headers.get("Content-Type"):
             parameters["ResponseContentType"] = headers.get("Content-Type")

pulpcore/constants.py

@@ -108,6 +108,7 @@
 # Storage-type mapped to storage-response-map
 STORAGE_RESPONSE_MAP = {
     "storages.backends.s3boto3.S3Boto3Storage": S3_RESPONSE_HEADER_MAP,
+    "storages.backends.s3.S3Storage": S3_RESPONSE_HEADER_MAP,
     "storages.backends.azure_storage.AzureStorage": AZURE_RESPONSE_HEADER_MAP,
     "storages.backends.gcloud.GoogleCloudStorage": GCS_RESPONSE_HEADER_MAP,
 }

pulpcore/content/handler.py

@@ -1021,7 +1021,10 @@ def _build_url(**kwargs):
             return FileResponse(path, headers=headers)
         elif not domain.redirect_to_object_storage:
             return ArtifactResponse(content_artifact.artifact, headers=headers)
-        elif domain.storage_class == "storages.backends.s3boto3.S3Boto3Storage":
+        elif domain.storage_class in (
+            "storages.backends.s3boto3.S3Boto3Storage",
+            "storages.backends.s3.S3Storage",
+        ):
             raise HTTPFound(_build_url(http_method=request.method), headers=headers)
         elif domain.storage_class in (
             "storages.backends.azure_storage.AzureStorage",

pulpcore/pytest_plugin.py

@@ -597,7 +597,7 @@ def _settings_factory(storage_class=None, storage_settings=None):
             pytest.skip("Domains not enabled")
         keys = dict()
         keys["pulpcore.app.models.storage.FileSystem"] = ["MEDIA_ROOT", "MEDIA_URL"]
-        keys["storages.backends.s3boto3.S3Boto3Storage"] = [
+        keys["storages.backends.s3.S3Storage"] = [
             "access_key",
             "secret_key",
             "endpoint_url",
@@ -606,6 +606,7 @@ def _settings_factory(storage_class=None, storage_settings=None):
             "region_name",
             "bucket_name",
         ]
+        keys["storages.backends.s3boto3.S3Boto3Storage"] = keys["storages.backends.s3.S3Storage"]
         keys["storages.backends.azure_storage.AzureStorage"] = [
             "AZURE_ACCOUNT_NAME",
             "AZURE_CONTAINER",

pulpcore/tests/functional/api/test_artifact_distribution.py

@@ -5,6 +5,7 @@
 
 OBJECT_STORAGES = (
     "storages.backends.s3boto3.S3Boto3Storage",
+    "storages.backends.s3.S3Storage",
     "storages.backends.azure_storage.AzureStorage",
     "storages.backends.gcloud.GoogleCloudStorage",
 )

pulpcore/tests/functional/api/test_crud_domains.py

@@ -190,6 +190,7 @@ def test_special_domain_creation(pulpcore_bindings, gen_object_with_cleanup, pul
         "pulpcore.app.models.storage.FileSystem",
         # "pulpcore.app.models.storage.PulpSFTPStorage",
         "storages.backends.s3boto3.S3Boto3Storage",
+        "storages.backends.s3.S3Storage",
         "storages.backends.azure_storage.AzureStorage",
         # "storages.backends.gcloud.GoogleCloudStorage",
     }
@@ -204,7 +205,7 @@ def test_special_domain_creation(pulpcore_bindings, gen_object_with_cleanup, pul
                 "key_filename": "/etc/pulp/certs/storage_id_ed25519",
             },
         },
-        "storages.backends.s3boto3.S3Boto3Storage": {
+        "storages.backends.s3.S3Storage": {
             "AWS_ACCESS_KEY_ID": "random",
             "AWS_SECRET_ACCESS_KEY": "random",
             "AWS_STORAGE_BUCKET_NAME": "pulp3",
@@ -227,6 +228,9 @@ def test_special_domain_creation(pulpcore_bindings, gen_object_with_cleanup, pul
             "GS_CUSTOM_ENDPOINT": "http://custom-endpoint",
         },
     }
+    storage_settings["storages.backends.s3boto3.S3Boto3Storage"] = storage_settings[
+        "storages.backends.s3.S3Storage"
+    ]
 
     installed_backends = []
     domain_names = set()
@@ -244,7 +248,8 @@ def test_special_domain_creation(pulpcore_bindings, gen_object_with_cleanup, pul
             assert e.status == 400
             assert "Backend is not installed on Pulp." in e.body
         else:
-            installed_backends.append(backend)
+            if backend != "storages.backends.s3boto3.S3Boto3Storage":
+                installed_backends.append(backend)
             domain_names.add(domain.name)
     # Try creating domains with correct settings
     for backend in installed_backends:
@@ -257,6 +262,7 @@ def test_special_domain_creation(pulpcore_bindings, gen_object_with_cleanup, pul
         domain_names.add(domain.name)
 
     # Try creating domains with incorrect settings
+    storage_types.remove("storages.backends.s3boto3.S3Boto3Storage")
     for backend in installed_backends:
         random_backend = random.choice(tuple(storage_types - {backend}))
         body = {

pulpcore/tests/unit/serializers/test_domain.py

@@ -30,6 +30,7 @@ def _no_validate_storage_backend(monkeypatch):
     params=[
         "pulpcore.app.models.storage.FileSystem",
         "storages.backends.s3boto3.S3Boto3Storage",
+        "storages.backends.s3.S3Storage",
         "storages.backends.azure_storage.AzureStorage",
     ]
 )
@@ -41,7 +42,10 @@ def storage_class(request):
 def serializer_class(storage_class):
     if storage_class == "pulpcore.app.models.storage.FileSystem":
         return FileSystemSettingsSerializer
-    elif storage_class == "storages.backends.s3boto3.S3Boto3Storage":
+    elif storage_class in (
+        "storages.backends.s3boto3.S3Boto3Storage",
+        "storages.backends.s3.S3Storage",
+    ):
         return AmazonS3SettingsSerializer
     elif storage_class == "storages.backends.azure_storage.AzureStorage":
         return AzureSettingsSerializer
@@ -51,7 +55,10 @@ def serializer_class(storage_class):
 def required_settings(storage_class):
     if storage_class == "pulpcore.app.models.storage.FileSystem":
         return {"location": "/var/lib/pulp/media/"}
-    elif storage_class == "storages.backends.s3boto3.S3Boto3Storage":
+    elif storage_class in (
+        "storages.backends.s3boto3.S3Boto3Storage",
+        "storages.backends.s3.S3Storage",
+    ):
         return {"access_key": "testing", "bucket_name": "test"}
     elif storage_class == "storages.backends.azure_storage.AzureStorage":
         return {"account_name": "test", "account_key": "secret", "azure_container": "test"}
@@ -60,7 +67,10 @@ def required_settings(storage_class):
 @pytest.fixture
 def extra_required_settings(storage_class):
     """For fields required in the serializer's validate, but not on the field itself."""
-    if storage_class == "storages.backends.s3boto3.S3Boto3Storage":
+    if storage_class in (
+        "storages.backends.s3boto3.S3Boto3Storage",
+        "storages.backends.s3.S3Storage",
+    ):
         return {"secret_key": "secret"}
     return {}
 
@@ -129,7 +139,10 @@ def test_using_setting_names(storage_class, serializer_class, all_settings):
 
 @pytest.mark.django_db
 def test_cloudfront_s3_storage_settings(storage_class, required_settings):
-    if storage_class != "storages.backends.s3boto3.S3Boto3Storage":
+    if storage_class not in (
+        "storages.backends.s3boto3.S3Boto3Storage",
+        "storages.backends.s3.S3Storage",
+    ):
         pytest.skip("This test only make sense when using S3 as storage backend.")
 
     domain = SimpleNamespace(storage_class=storage_class, **MIN_DOMAIN_SETTINGS)