[MISC] Conditional checksum calculation (#901)

* Conditional checksum calculation

* Converge s3 resource creation

* Tactically deployed sleep

* Early fail

Author: dragomirp

poetry.lock

@@ -8,7 +8,7 @@ requires-poetry = ">=2.0.0"
 [tool.poetry.dependencies]
 python = "^3.10"
 ops = "^2.18.1"
-boto3 = "^1.35.99"
+boto3 = "^1.37.22"
 pgconnstr = "^1.0.1"
 requests = "^2.32.3"
 tenacity = "^9.0.0"

pyproject.toml

@@ -12,7 +12,7 @@
 from datetime import datetime, timezone
 from io import BytesIO
 
-import boto3 as boto3
+import boto3
 import botocore
 from botocore.exceptions import ClientError
 from charms.data_platform_libs.v0.s3 import CredentialsChangedEvent, S3Requirer
@@ -88,6 +88,23 @@ def _tls_ca_chain_filename(self) -> str:
             return f"{self.charm._storage_path}/pgbackrest-tls-ca-chain.crt"
         return ""
 
+    def _get_s3_session_resource(self, s3_parameters: dict):
+        session = boto3.session.Session(
+            aws_access_key_id=s3_parameters["access-key"],
+            aws_secret_access_key=s3_parameters["secret-key"],
+            region_name=s3_parameters["region"],
+        )
+        return session.resource(
+            "s3",
+            endpoint_url=self._construct_endpoint(s3_parameters),
+            verify=(self._tls_ca_chain_filename or None),
+            config=botocore.client.Config(
+                # https://github.com/boto/boto3/issues/4400#issuecomment-2600742103
+                request_checksum_calculation="when_required",
+                response_checksum_validation="when_required",
+            ),
+        )
+
     def _are_backup_settings_ok(self) -> tuple[bool, str | None]:
         """Validates whether backup settings are OK."""
         if self.model.get_relation(self.relation_name) is None:
@@ -227,18 +244,9 @@ def _create_bucket_if_not_exists(self) -> None:
 
         bucket_name = s3_parameters["bucket"]
         region = s3_parameters.get("region")
-        session = boto3.session.Session(
-            aws_access_key_id=s3_parameters["access-key"],
-            aws_secret_access_key=s3_parameters["secret-key"],
-            region_name=s3_parameters["region"],
-        )
 
         try:
-            s3 = session.resource(
-                "s3",
-                endpoint_url=self._construct_endpoint(s3_parameters),
-                verify=(self._tls_ca_chain_filename or None),
-            )
+            s3 = self._get_s3_session_resource(s3_parameters)
         except ValueError as e:
             logger.exception("Failed to create a session '%s' in region=%s.", bucket_name, region)
             raise e
@@ -1316,17 +1324,8 @@ def _upload_content_to_s3(
         processed_s3_path = os.path.join(s3_parameters["path"], s3_path).lstrip("/")
         try:
             logger.info(f"Uploading content to bucket={bucket_name}, path={processed_s3_path}")
-            session = boto3.session.Session(
-                aws_access_key_id=s3_parameters["access-key"],
-                aws_secret_access_key=s3_parameters["secret-key"],
-                region_name=s3_parameters["region"],
-            )
 
-            s3 = session.resource(
-                "s3",
-                endpoint_url=self._construct_endpoint(s3_parameters),
-                verify=(self._tls_ca_chain_filename or None),
-            )
+            s3 = self._get_s3_session_resource(s3_parameters)
             bucket = s3.Bucket(bucket_name)
 
             with tempfile.NamedTemporaryFile() as temp_file:
@@ -1359,16 +1358,7 @@ def _read_content_from_s3(self, s3_path: str, s3_parameters: dict) -> str | None
         processed_s3_path = os.path.join(s3_parameters["path"], s3_path).lstrip("/")
         try:
             logger.info(f"Reading content from bucket={bucket_name}, path={processed_s3_path}")
-            session = boto3.session.Session(
-                aws_access_key_id=s3_parameters["access-key"],
-                aws_secret_access_key=s3_parameters["secret-key"],
-                region_name=s3_parameters["region"],
-            )
-            s3 = session.resource(
-                "s3",
-                endpoint_url=self._construct_endpoint(s3_parameters),
-                verify=(self._tls_ca_chain_filename or None),
-            )
+            s3 = self._get_s3_session_resource(s3_parameters)
             bucket = s3.Bucket(bucket_name)
             with BytesIO() as buf:
                 bucket.download_fileobj(processed_s3_path, buf)

src/backups.py

@@ -72,6 +72,7 @@ async def test_backup_gcp(ops_test: OpsTest, charm, gcp_cloud_configs: tuple[dic
     )
 
 
+@pytest.mark.abort_on_fail
 async def test_restore_on_new_cluster(
     ops_test: OpsTest, charm, gcp_cloud_configs: tuple[dict, dict]
 ) -> None: