[DPE-5908] Support old bucket endpoints format (#603)

Author: sinclert-canonical

lib/charms/mysql/v0/s3_helpers.py

@@ -21,6 +21,7 @@
 from typing import Dict, List, Tuple
 
 import boto3
+import botocore
 
 logger = logging.getLogger(__name__)
 
@@ -32,13 +33,36 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 9
+LIBPATCH = 10
 
 # botocore/urllib3 clutter the logs when on debug
 logging.getLogger("botocore").setLevel(logging.WARNING)
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 
 
+def _construct_endpoint(s3_parameters: dict) -> str:
+    """Construct the S3 service endpoint using the region.
+
+    This is needed when the provided endpoint is from AWS, and it doesn't contain the region.
+    """
+    # Use the provided endpoint if a region is not needed.
+    endpoint = s3_parameters["endpoint"]
+
+    # Load endpoints data.
+    loader = botocore.loaders.create_loader()
+    data = loader.load_data("endpoints")
+
+    # Construct the endpoint using the region.
+    resolver = botocore.regions.EndpointResolver(data)
+    endpoint_data = resolver.construct_endpoint("s3", s3_parameters["region"])
+
+    # Use the built endpoint if it is an AWS endpoint.
+    if endpoint_data and endpoint.endswith(endpoint_data["dnsSuffix"]):
+        endpoint = f"{endpoint.split('://')[0]}://{endpoint_data['hostname']}"
+
+    return endpoint
+
+
 def upload_content_to_s3(content: str, content_path: str, s3_parameters: Dict) -> bool:
     """Uploads the provided contents to the provided S3 bucket.
 
@@ -67,11 +91,7 @@ def upload_content_to_s3(content: str, content_path: str, s3_parameters: Dict) -
             ca_file.flush()
             verif = ca_file.name
 
-        s3 = session.resource(
-            "s3",
-            endpoint_url=s3_parameters["endpoint"],
-            verify=verif,
-        )
+        s3 = session.resource("s3", endpoint_url=_construct_endpoint(s3_parameters), verify=verif)
 
         bucket = s3.Bucket(s3_parameters["bucket"])
 

tests/unit/test_s3_helpers.py

@@ -12,7 +12,7 @@ def setUp(self) -> None:
         self.s3_parameters = {
             "access-key": "AK",
             "secret-key": "SK",
-            "region": "AS-1",
+            "region": "us-east-1",
             "bucket": "balde",
             "endpoint": "http://localhost:9000",
         }
@@ -44,16 +44,50 @@ def test_upload_content_with_ca_chain(self, mock_boto):
         mock_session.resource.return_value = mock_resource
         mock_resource.Bucket.return_value = mock_bucket
 
-        s3_parameters = {
-            "access-key": "AK",
-            "secret-key": "SK",
-            "region": "AS-1",
-            "bucket": "balde",
-            "endpoint": "http://localhost:9000",
-            "tls-ca-chain": ["Zm9vYmFy"],
-        }
+        s3_parameters = self.s3_parameters
+        s3_parameters["tls-ca-chain"] = ["Zm9vYmFy"]
 
         upload_content_to_s3("content", "key", s3_parameters)
 
         mock_bucket.upload_file.assert_called_once()
         mock_session.resource.assert_called_once()
+
+    @patch("lib.charms.mysql.v0.s3_helpers.boto3")
+    def test_upload_content_with_new_bucket_endpoint(self, mock_boto):
+        mock_session = MagicMock()
+        mock_resource = MagicMock()
+        mock_bucket = MagicMock()
+
+        mock_boto.session.Session.return_value = mock_session
+        mock_session.resource.return_value = mock_resource
+        mock_resource.Bucket.return_value = mock_bucket
+
+        s3_parameters = self.s3_parameters
+        s3_parameters["endpoint"] = "https://s3.us-east-1.amazonaws.com"
+
+        upload_content_to_s3("content", "key", s3_parameters)
+
+        mock_bucket.upload_file.assert_called_once()
+        mock_session.resource.assert_called_with(
+            "s3", endpoint_url="https://s3.us-east-1.amazonaws.com", verify=True
+        )
+
+    @patch("lib.charms.mysql.v0.s3_helpers.boto3")
+    def test_upload_content_with_old_bucket_endpoint(self, mock_boto):
+        mock_session = MagicMock()
+        mock_resource = MagicMock()
+        mock_bucket = MagicMock()
+
+        mock_boto.session.Session.return_value = mock_session
+        mock_session.resource.return_value = mock_resource
+        mock_resource.Bucket.return_value = mock_bucket
+
+        s3_parameters = self.s3_parameters
+        s3_parameters["endpoint"] = "https://s3.amazonaws.com"
+
+        upload_content_to_s3("content", "key", s3_parameters)
+
+        mock_bucket.upload_file.assert_called_once()
+        mock_session.resource.assert_called_with(
+            "s3", endpoint_url="https://s3.us-east-1.amazonaws.com", verify=True
+        )