Merge branch 'master' into docs-connect-release

Author: drernie

lambdas/access_counts/CHANGELOG.md

@@ -17,6 +17,8 @@ where verb is one of
 
 ## Changes
 
+- [Added] Make `s3.copy()` chunk size and concurrency configurable via env vars ([#4746](https://github.com/quiltdata/quilt/pull/4746))
+- [Added] Bundle `awscrt` via `boto3[crt]` for improved S3 transfer performance ([#4746](https://github.com/quiltdata/quilt/pull/4746))
 - [Changed] Migrate to proper package structure ([#4618](https://github.com/quiltdata/quilt/pull/4618))
 - [Changed] Switch to uv ([#4618](https://github.com/quiltdata/quilt/pull/4618))
 - [Changed] Upgrade to Python 3.13 ([#4618](https://github.com/quiltdata/quilt/pull/4618))

lambdas/access_counts/pyproject.toml

@@ -3,7 +3,7 @@ name = "t4_lambda_access_counts"
 version = "0.1.0"
 requires-python = "==3.13.*"
 dependencies = [
-    "boto3~=1.40",
+    "boto3[crt]~=1.40",
     "t4_lambda_shared",
 ]
 

lambdas/access_counts/pytest.ini

@@ -7,3 +7,5 @@ env =
     CLOUDTRAIL_BUCKET = cloudtrail-bucket
     QUERY_RESULT_BUCKET = results-bucket
     ACCESS_COUNTS_OUTPUT_DIR = AccessCounts
+    S3_COPY_CHUNKSIZE_MIB = 128
+    S3_COPY_MAX_CONCURRENCY = 100

lambdas/access_counts/src/t4_lambda_access_counts/index.py

@@ -9,6 +9,7 @@
 from datetime import datetime, timedelta, timezone
 
 import boto3
+from boto3.s3.transfer import TransferConfig
 
 from t4_lambda_shared.utils import sql_escape
 
@@ -20,6 +21,14 @@
 # Directory where the summary files will be stored.
 ACCESS_COUNTS_OUTPUT_DIR = os.environ['ACCESS_COUNTS_OUTPUT_DIR']
 
+MiB = 1024 * 1024
+S3_COPY_CHUNKSIZE = int(os.environ['S3_COPY_CHUNKSIZE_MIB']) * MiB
+S3_COPY_CONFIG = TransferConfig(
+    multipart_chunksize=S3_COPY_CHUNKSIZE,
+    multipart_threshold=S3_COPY_CHUNKSIZE,
+    max_concurrency=int(os.environ['S3_COPY_MAX_CONCURRENCY']),
+)
+
 # A temporary directory where Athena query results will be written.
 QUERY_TEMP_DIR = 'AthenaQueryResults'
 
@@ -453,5 +462,6 @@ def handler(event, context):
                 Key=src_key
             ),
             Bucket=QUERY_RESULT_BUCKET,
-            Key=dest_key
+            Key=dest_key,
+            Config=S3_COPY_CONFIG,
         )