Add support for S3 regions

ami/main/migrations/0075_s3storagesource_region.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.10 on 2025-09-26 08:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("main", "0074_taxon_cover_image_credit_taxon_cover_image_url_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="s3storagesource",
+            name="region",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

ami/main/models.py

@@ -1394,6 +1394,7 @@ class S3StorageSource(BaseModel):
 
     name = models.CharField(max_length=255)
     bucket = models.CharField(max_length=255)
+    region = models.CharField(max_length=255, null=True, blank=True)
     prefix = models.CharField(max_length=255, blank=True)
     access_key = models.TextField()
     secret_key = models.TextField()
@@ -1413,6 +1414,7 @@ class S3StorageSource(BaseModel):
     def config(self) -> ami.utils.s3.S3Config:
         return ami.utils.s3.S3Config(
             bucket_name=self.bucket,
+            region=self.region,
             prefix=self.prefix,
             access_key_id=self.access_key,
             secret_access_key=self.secret_key,

ami/tests/fixtures/storage.py

@@ -15,6 +15,7 @@
     access_key_id=settings.S3_TEST_KEY,
     secret_access_key=settings.S3_TEST_SECRET,
     bucket_name=settings.S3_TEST_BUCKET,
+    region=settings.S3_TEST_REGION,
     prefix="test_prefix",
     public_base_url=f"http://minio:9000/{settings.S3_TEST_BUCKET}/test_prefix",
     # public_base_url="http://minio:9001",

ami/utils/s3.py

@@ -10,7 +10,7 @@
 from dataclasses import dataclass
 
 import boto3
-import boto3.resources.base
+import boto3.session
 import botocore
 import botocore.config
 import botocore.exceptions
@@ -37,6 +37,7 @@ class S3Config:
     secret_access_key: str
     bucket_name: str
     prefix: str
+    region: str | None = None
     public_base_url: str | None = None
 
     sensitive_fields = ["access_key_id", "secret_access_key"]
@@ -94,26 +95,36 @@ def get_session(config: S3Config) -> boto3.session.Session:
     session = boto3.Session(
         aws_access_key_id=config.access_key_id,
         aws_secret_access_key=config.secret_access_key,
+        region_name=config.region,
     )
     return session
 
 
 def get_s3_client(config: S3Config) -> S3Client:
     session = get_session(config)
+
+    # Always use signature version 4
+    boto_config = botocore.config.Config(signature_version="s3v4")
+
     if config.endpoint_url:
         client = session.client(
             service_name="s3",
             endpoint_url=config.endpoint_url,
             aws_access_key_id=config.access_key_id,
             aws_secret_access_key=config.secret_access_key,
-            config=botocore.config.Config(signature_version="s3v4"),
+            region_name=config.region,
+            config=boto_config,
         )
     else:
         client = session.client(
             service_name="s3",
             aws_access_key_id=config.access_key_id,
             aws_secret_access_key=config.secret_access_key,
+            region_name=config.region,
+            config=boto_config,
         )
+
+    client = typing.cast(S3Client, client)
     return client
 
 
@@ -124,6 +135,7 @@ def get_resource(config: S3Config) -> S3ServiceResource:
         endpoint_url=config.endpoint_url,
         # api_version="s3v4",
     )
+    s3 = typing.cast(S3ServiceResource, s3)
     return s3
 
 
@@ -584,7 +596,9 @@ def read_image(config: S3Config, key: str) -> PIL.Image.Image:
     obj = bucket.Object(key)
     logger.info(f"Fetching image {key} from S3")
     try:
-        img = PIL.Image.open(obj.get()["Body"])
+        # StreamingBody inherits from io.IOBase, but type checkers don't see that
+        fp = obj.get()["Body"]
+        img = PIL.Image.open(fp)  # type: ignore[arg-type]
     except PIL.UnidentifiedImageError:
         logger.error(f"Could not read image {key}")
         raise
@@ -677,6 +691,7 @@ def test():
         bucket_name="test",
         prefix="",
         public_base_url="http://minio:9000/test",
+        region=None,
     )
 
     projects = list_projects(config)

config/settings/base.py

@@ -449,6 +449,7 @@
 S3_TEST_KEY = env("MINIO_ROOT_USER", default=None)  # type: ignore[no-untyped-call]
 S3_TEST_SECRET = env("MINIO_ROOT_PASSWORD", default=None)  # type: ignore[no-untyped-call]
 S3_TEST_BUCKET = env("MINIO_TEST_BUCKET", default="ami-test")  # type: ignore[no-untyped-call]
+S3_TEST_REGION = env("MINIO_REGION", default=None)  # type: ignore[no-untyped-call]
 
 
 # Default processing service settings