Skip to content

Commit 893ee0e

Browse files
fix unit test
1 parent 75521c6 commit 893ee0e

File tree

4 files changed

+48
-26
lines changed

4 files changed

+48
-26
lines changed

.secrets.baseline

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,9 +178,9 @@
178178
"filename": "tests/test_s3_endpoint.py",
179179
"hashed_secret": "08d2e98e6754af941484848930ccbaddfefe13d6",
180180
"is_verified": false,
181-
"line_number": 77
181+
"line_number": 82
182182
}
183183
]
184184
},
185-
"generated_at": "2026-03-02T20:27:56Z"
185+
"generated_at": "2026-03-03T22:13:17Z"
186186
}

gen3workflow/routes/s3.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -284,8 +284,6 @@ async def s3_endpoint(path: str, request: Request):
284284
# We know this is an UploadPart or CompleteMultipartUpload request if it includes the
285285
# uploadId query parameter.
286286
query_params = dict(request.query_params)
287-
if "uploadId" in query_params:
288-
raise Exception("multipart")
289287
if (
290288
config["KMS_ENCRYPTION_ENABLED"]
291289
and request.method in ["PUT", "POST"]

tests/conftest.py

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -226,10 +226,33 @@ def mock_tes_server_request_function(
226226
return httpx.Response(status_code=status_code, json=out, text=text)
227227

228228

229+
def mock_aws_s3_request_function(url: str):
230+
"""
231+
Mock responses from AWS S3
232+
"""
233+
resp_xml = MOCKED_S3_RESPONSE_XML
234+
headers = {"content-type": "application/xml"}
235+
236+
# multipart upload special case:
237+
if "test_s3_upload_file_multipart.txt" in url:
238+
upload_id = "test-upload-id"
239+
# "InitiateMultipartUploadResult" with "UploadId"
240+
resp_xml = f"""<?xml version="1.0" encoding="UTF-8"?>\n<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Bucket>gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}</Bucket><Key>test_s3_chunk_upload.txt</Key><UploadId>{upload_id}</UploadId></InitiateMultipartUploadResult>"""
241+
if f"?uploadId={upload_id}&partNumber=" in url:
242+
headers["etag"] = "test-etag"
243+
244+
return httpx.Response(
245+
status_code=200,
246+
text=resp_xml,
247+
headers=headers,
248+
)
249+
250+
229251
# making these functions into mocks allows tests to check the requests that were made, for
230252
# example: `mock_tes_server_request.assert_called_with(...)`
231253
mock_tes_server_request = MagicMock(side_effect=mock_tes_server_request_function)
232254
mock_arborist_request = MagicMock(side_effect=mock_arborist_request_function)
255+
mock_aws_s3_request = MagicMock(side_effect=mock_aws_s3_request_function)
233256

234257

235258
@pytest_asyncio.fixture(scope="function", autouse=True)
@@ -316,20 +339,7 @@ async def handle_request(request: Request):
316339
f"https://gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}.s3.{config['USER_BUCKETS_REGION']}.amazonaws.com"
317340
):
318341
# mock calls to AWS S3
319-
resp_xml = MOCKED_S3_RESPONSE_XML
320-
headers = {"content-type": "application/xml"}
321-
# multipart upload special case:
322-
if "test_s3_upload_file_multipart.txt" in url:
323-
upload_id = "test-upload-id"
324-
# "InitiateMultipartUploadResult" with "UploadId"
325-
resp_xml = f"""<?xml version="1.0" encoding="UTF-8"?>\n<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Bucket>gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}</Bucket><Key>test_s3_chunk_upload.txt</Key><UploadId>{upload_id}</UploadId></InitiateMultipartUploadResult>"""
326-
if f"?uploadId={upload_id}&partNumber=" in url:
327-
headers["etag"] = "test-etag"
328-
mocked_response = httpx.Response(
329-
status_code=200,
330-
text=resp_xml,
331-
headers=headers,
332-
)
342+
mocked_response = mock_aws_s3_request(url)
333343

334344
if mocked_response is not None:
335345
print(f"Mocking request '{request.method} {url}'")

tests/test_s3_endpoint.py

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,12 @@
88
from fastapi import HTTPException
99
import pytest
1010

11-
from conftest import MOCKED_S3_RESPONSE_DICT, TEST_USER_ID, TEST_USER_TOKEN
11+
from conftest import (
12+
MOCKED_S3_RESPONSE_DICT,
13+
TEST_USER_ID,
14+
TEST_USER_TOKEN,
15+
mock_aws_s3_request,
16+
)
1217
from gen3workflow.config import config
1318
from gen3workflow.routes.s3 import (
1419
set_access_token_and_get_user_id,
@@ -350,24 +355,33 @@ def test_s3_upload_file(s3_client, access_token_patcher, multipart):
350355
Test that the boto3 `upload_file` function works with the `/s3` endpoint, both for a small
351356
file uploaded in 1 part and for a large file uploaded in multiple parts.
352357
"""
358+
bucket_name = f"gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}"
359+
object_key = f"test_s3_upload_file{'_multipart' if multipart else ''}.txt"
360+
353361
with patch(
354362
"gen3workflow.aws_utils.get_existing_kms_key_for_bucket",
355363
lambda _: ("test_kms_key_alias", "test_kms_key_arn"),
356364
):
357-
with tempfile.NamedTemporaryFile(mode="w+t", delete=True) as file_to_upload:
358-
file_to_upload.write("Test file contents\n")
365+
with tempfile.NamedTemporaryFile(delete=True) as file_to_upload:
366+
file_to_upload.write(b"A" * (6 * 1024 * 1024)) # create a 6MB file
367+
file_to_upload.flush()
359368
s3_client.upload_file(
360369
file_to_upload.name,
361-
f"gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}",
362-
f"test_s3_upload_file{'_multipart' if multipart else ''}.txt",
370+
bucket_name,
371+
object_key,
363372
# to test a multipart upload, set the part size to 1 to force splitting the file
364373
# into multiple parts:
365-
# TODO this doesn't work
366-
Config=boto3.s3.transfer.TransferConfig(
367-
multipart_threshold=1 if multipart else 9999
374+
Config=(
375+
boto3.s3.transfer.TransferConfig(multipart_threshold=1)
376+
if multipart
377+
else None
368378
),
369379
)
370380

381+
mock_aws_s3_request.assert_called_with(
382+
f"https://{bucket_name}.s3.us-east-1.amazonaws.com/{object_key}{'?uploadId=test-upload-id' if multipart else ''}"
383+
)
384+
371385

372386
def test_chunked_to_non_chunked_body():
373387
"""

0 commit comments

Comments
 (0)