Skip to content

Commit 2566b29

Browse files
Copilotchrisburr
Copilot
and
chrisburr
authored
Implement delete_objects method (#18)
* Initial plan * Implement delete_objects method for sync and async clients Add `delete_objects` method to both `Client` and `AsyncClient` that performs S3 multi-object delete via POST request with XML payload. Changes: - presigner.py: Add optional `query_string` param to `sign_request_headers` - _base.py: Add `_prepare_delete_objects` and `_parse_delete_objects_response` - client.py: Add POST support to `_execute_request` and `delete_objects` method - aio/client.py: Add POST support to `_execute_request` and `delete_objects` method - tests: Add sync and async tests for delete_objects Co-authored-by: chrisburr <5220533+chrisburr@users.noreply.github.com> * Fix XML injection vulnerability in delete_objects XML body construction Co-authored-by: chrisburr <5220533+chrisburr@users.noreply.github.com> * Fix pre-commit formatting and add delete_objects benchmarks Fix ruff format issues (line length) in source files and add delete_objects benchmarks to all 4 benchmark test files: - test_benchmark.py (sync) - test_benchmark_cm.py (sync context manager) - test_benchmark_aio.py (async) - test_benchmark_aio_cm.py (async context manager) Co-authored-by: chrisburr <5220533+chrisburr@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: chrisburr <5220533+chrisburr@users.noreply.github.com>
1 parent 403423f commit 2566b29

File tree

10 files changed

+766
-2
lines changed

10 files changed

+766
-2
lines changed

‎benchmark_tests/test_benchmark.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,3 +391,95 @@ def run_custom(n: int):
391391
result_file.write_text(json.dumps(results, indent=2))
392392

393393
print("=" * 60)
394+
395+
396+
def test_delete_objects_perf_sync(rustfs_server, test_results_dir):
397+
"""Compare performance of boto3 vs signurlarity for delete_objects.
398+
399+
This benchmark compares boto3's delete_objects with the signurlarity
400+
implementation that uses httpx with AWS Signature V4.
401+
"""
402+
py_vers = sys.version_info
403+
test_dir = test_results_dir / Path("test_delete_objects_perf")
404+
os.makedirs(test_dir, exist_ok=True)
405+
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
406+
407+
bucket = "perf-delete-objects"
408+
num_keys = 10
409+
410+
boto_client = boto3.client("s3", **rustfs_server)
411+
light_client = Client(**rustfs_server)
412+
413+
# Create the bucket for testing
414+
boto_client.create_bucket(Bucket=bucket)
415+
416+
iterations = 10
417+
418+
def _populate(prefix: str):
419+
keys = [f"{prefix}-{i}.txt" for i in range(num_keys)]
420+
for k in keys:
421+
boto_client.put_object(Bucket=bucket, Key=k, Body=b"data")
422+
return keys
423+
424+
# Warm-up
425+
for i in range(5):
426+
keys = _populate(f"warmup-boto-{i}")
427+
boto_client.delete_objects(
428+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
429+
)
430+
431+
for i in range(5):
432+
keys = _populate(f"warmup-light-{i}")
433+
light_client.delete_objects(
434+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
435+
)
436+
437+
def run_boto(n: int):
438+
for i in range(n):
439+
keys = _populate(f"bench-boto-{i}")
440+
boto_client.delete_objects(
441+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
442+
)
443+
444+
def run_custom(n: int):
445+
for i in range(n):
446+
keys = _populate(f"bench-light-{i}")
447+
light_client.delete_objects(
448+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
449+
)
450+
451+
t_boto = _timeit(run_boto, iterations)
452+
t_custom = _timeit(run_custom, iterations)
453+
454+
light_client.close()
455+
results = {
456+
"python_version": f"{py_vers.major}.{py_vers.minor}",
457+
"tested_method": "delete_objects_sync",
458+
"iterations": iterations,
459+
"boto_total": t_boto,
460+
"signurlarity_total": t_custom,
461+
"boto_ops": iterations / t_boto,
462+
"signurlarity_ops": iterations / t_custom,
463+
"speedup": t_boto / t_custom,
464+
}
465+
466+
print("\n" + "=" * 60)
467+
print("DELETE OBJECTS BENCHMARK")
468+
print("=" * 60)
469+
print(
470+
f"boto3 delete_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)"
471+
)
472+
print(
473+
f"signurlarity delete_objects: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)"
474+
)
475+
if t_custom > 0:
476+
speedup = t_boto / t_custom
477+
print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x")
478+
if speedup > 1:
479+
print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!")
480+
else:
481+
print(f"boto3 is {1 / speedup:.2f}x faster")
482+
483+
result_file.write_text(json.dumps(results, indent=2))
484+
485+
print("=" * 60)

‎benchmark_tests/test_benchmark_aio.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -387,3 +387,97 @@ async def run_custom(n: int):
387387
result_file.write_text(json.dumps(results, indent=2))
388388

389389
print("=" * 60)
390+
391+
392+
@pytest.mark.asyncio
393+
async def test_delete_objects_perf_aio(rustfs_server, test_results_dir):
394+
"""Compare performance of boto3 vs signurlarity async for delete_objects.
395+
396+
This benchmark tests the async implementation's delete_objects functionality.
397+
"""
398+
py_vers = sys.version_info
399+
test_dir = test_results_dir / Path("test_delete_objects_perf_aio")
400+
os.makedirs(test_dir, exist_ok=True)
401+
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
402+
403+
bucket = "perf-delete-objects"
404+
num_keys = 10
405+
406+
async_light_client = AsyncClient(**rustfs_server)
407+
session = get_session()
408+
async with session.create_client(
409+
"s3", **rustfs_server, config=Config(signature_version="s3v4")
410+
) as boto_client:
411+
# Create the bucket for testing
412+
await async_light_client.create_bucket(Bucket=bucket)
413+
414+
iterations = 10
415+
416+
async def _populate(prefix: str):
417+
keys = [f"{prefix}-{i}.txt" for i in range(num_keys)]
418+
for k in keys:
419+
await boto_client.put_object(Bucket=bucket, Key=k, Body=b"data")
420+
return keys
421+
422+
# Warm-up
423+
for i in range(5):
424+
keys = await _populate(f"warmup-boto-{i}")
425+
await boto_client.delete_objects(
426+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
427+
)
428+
429+
for i in range(5):
430+
keys = await _populate(f"warmup-light-{i}")
431+
await async_light_client.delete_objects(
432+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
433+
)
434+
435+
async def run_boto(n: int):
436+
for i in range(n):
437+
keys = await _populate(f"bench-boto-{i}")
438+
await boto_client.delete_objects(
439+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
440+
)
441+
442+
async def run_custom(n: int):
443+
for i in range(n):
444+
keys = await _populate(f"bench-light-{i}")
445+
await async_light_client.delete_objects(
446+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
447+
)
448+
449+
t_boto = await _timeit_async_helper(run_boto, iterations)
450+
t_custom = await _timeit_async_helper(run_custom, iterations)
451+
452+
await async_light_client.close()
453+
results = {
454+
"python_version": f"{py_vers.major}.{py_vers.minor}",
455+
"tested_method": "delete_objects_aio",
456+
"iterations": iterations,
457+
"boto_total": t_boto,
458+
"signurlarity_total": t_custom,
459+
"boto_ops": iterations / t_boto,
460+
"signurlarity_ops": iterations / t_custom,
461+
"speedup": t_boto / t_custom,
462+
}
463+
464+
print("\n" + "=" * 60)
465+
print("DELETE OBJECTS BENCHMARK (ASYNC)")
466+
print("=" * 60)
467+
print(
468+
f"boto3 delete_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)"
469+
)
470+
print(
471+
f"signurlarity delete_objects (async): {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)"
472+
)
473+
if t_custom > 0:
474+
speedup = t_boto / t_custom
475+
print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x")
476+
if speedup > 1:
477+
print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!")
478+
else:
479+
print(f"boto3 is {1 / speedup:.2f}x faster")
480+
481+
result_file.write_text(json.dumps(results, indent=2))
482+
483+
print("=" * 60)

‎benchmark_tests/test_benchmark_aio_cm.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,3 +381,99 @@ async def run_custom(n: int):
381381
result_file.write_text(json.dumps(results, indent=2))
382382

383383
print("=" * 60)
384+
385+
386+
@pytest.mark.asyncio
387+
async def test_delete_objects_perf_aio_cm(rustfs_server, test_results_dir):
388+
"""Compare performance of boto3 vs signurlarity async for delete_objects.
389+
390+
This benchmark tests the async implementation's delete_objects functionality.
391+
"""
392+
py_vers = sys.version_info
393+
test_dir = test_results_dir / Path("test_delete_objects_perf_aio_cm")
394+
os.makedirs(test_dir, exist_ok=True)
395+
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
396+
397+
bucket = "perf-delete-objects"
398+
num_keys = 10
399+
400+
session = get_session()
401+
async with session.create_client(
402+
"s3", **rustfs_server, config=Config(signature_version="s3v4")
403+
) as boto_client:
404+
async with AsyncClient(**rustfs_server) as async_light_client:
405+
# Create the bucket for testing
406+
await async_light_client.create_bucket(Bucket=bucket)
407+
408+
iterations = 10
409+
410+
async def _populate(prefix: str):
411+
keys = [f"{prefix}-{i}.txt" for i in range(num_keys)]
412+
for k in keys:
413+
await boto_client.put_object(Bucket=bucket, Key=k, Body=b"data")
414+
return keys
415+
416+
# Warm-up
417+
for i in range(5):
418+
keys = await _populate(f"warmup-boto-{i}")
419+
await boto_client.delete_objects(
420+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
421+
)
422+
423+
for i in range(5):
424+
keys = await _populate(f"warmup-light-{i}")
425+
await async_light_client.delete_objects(
426+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
427+
)
428+
429+
async def run_boto(n: int):
430+
for i in range(n):
431+
keys = await _populate(f"bench-boto-{i}")
432+
await boto_client.delete_objects(
433+
Bucket=bucket,
434+
Delete={"Objects": [{"Key": k} for k in keys]},
435+
)
436+
437+
async def run_custom(n: int):
438+
for i in range(n):
439+
keys = await _populate(f"bench-light-{i}")
440+
await async_light_client.delete_objects(
441+
Bucket=bucket,
442+
Delete={"Objects": [{"Key": k} for k in keys]},
443+
)
444+
445+
t_boto = await _timeit_async_helper(run_boto, iterations)
446+
t_custom = await _timeit_async_helper(run_custom, iterations)
447+
448+
results = {
449+
"python_version": f"{py_vers.major}.{py_vers.minor}",
450+
"tested_method": "delete_objects_aio_cm",
451+
"iterations": iterations,
452+
"boto_total": t_boto,
453+
"signurlarity_total": t_custom,
454+
"boto_ops": iterations / t_boto,
455+
"signurlarity_ops": iterations / t_custom,
456+
"speedup": t_boto / t_custom,
457+
}
458+
459+
print("\n" + "=" * 60)
460+
print("DELETE OBJECTS BENCHMARK (ASYNC CM)")
461+
print("=" * 60)
462+
print(
463+
f"boto3 delete_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)"
464+
)
465+
print(
466+
f"signurlarity delete_objects (async cm): {t_custom:.4f}s"
467+
f" for {iterations} ops ({iterations / t_custom:.0f} ops/s)"
468+
)
469+
if t_custom > 0:
470+
speedup = t_boto / t_custom
471+
print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x")
472+
if speedup > 1:
473+
print(f"✓ Signurlarity async implementation is {speedup:.2f}x FASTER!")
474+
else:
475+
print(f"boto3 is {1 / speedup:.2f}x faster")
476+
477+
result_file.write_text(json.dumps(results, indent=2))
478+
479+
print("=" * 60)

‎benchmark_tests/test_benchmark_cm.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,3 +391,93 @@ def run_custom(n: int):
391391
result_file.write_text(json.dumps(results, indent=2))
392392

393393
print("=" * 60)
394+
395+
396+
def test_delete_objects_perf_sync_cm(rustfs_server, test_results_dir):
397+
"""Compare performance of boto3 vs signurlarity for delete_objects.
398+
399+
This benchmark compares boto3's delete_objects with the signurlarity
400+
implementation that uses httpx with AWS Signature V4.
401+
"""
402+
py_vers = sys.version_info
403+
test_dir = test_results_dir / Path("test_delete_objects_perf_sync_cm")
404+
os.makedirs(test_dir, exist_ok=True)
405+
result_file: Path = test_dir / Path(f"run_{py_vers.major}.{py_vers.minor}.json")
406+
407+
bucket = "perf-delete-objects"
408+
num_keys = 10
409+
410+
boto_client = boto3.client("s3", **rustfs_server)
411+
with Client(**rustfs_server) as light_client:
412+
# Create the bucket for testing
413+
boto_client.create_bucket(Bucket=bucket)
414+
415+
iterations = 10
416+
417+
def _populate(prefix: str):
418+
keys = [f"{prefix}-{i}.txt" for i in range(num_keys)]
419+
for k in keys:
420+
boto_client.put_object(Bucket=bucket, Key=k, Body=b"data")
421+
return keys
422+
423+
# Warm-up
424+
for i in range(5):
425+
keys = _populate(f"warmup-boto-{i}")
426+
boto_client.delete_objects(
427+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
428+
)
429+
430+
for i in range(5):
431+
keys = _populate(f"warmup-light-{i}")
432+
light_client.delete_objects(
433+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
434+
)
435+
436+
def run_boto(n: int):
437+
for i in range(n):
438+
keys = _populate(f"bench-boto-{i}")
439+
boto_client.delete_objects(
440+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
441+
)
442+
443+
def run_custom(n: int):
444+
for i in range(n):
445+
keys = _populate(f"bench-light-{i}")
446+
light_client.delete_objects(
447+
Bucket=bucket, Delete={"Objects": [{"Key": k} for k in keys]}
448+
)
449+
450+
t_boto = _timeit(run_boto, iterations)
451+
t_custom = _timeit(run_custom, iterations)
452+
453+
results = {
454+
"python_version": f"{py_vers.major}.{py_vers.minor}",
455+
"tested_method": "delete_objects_sync_cm",
456+
"iterations": iterations,
457+
"boto_total": t_boto,
458+
"signurlarity_total": t_custom,
459+
"boto_ops": iterations / t_boto,
460+
"signurlarity_ops": iterations / t_custom,
461+
"speedup": t_boto / t_custom,
462+
}
463+
464+
print("\n" + "=" * 60)
465+
print("DELETE OBJECTS BENCHMARK")
466+
print("=" * 60)
467+
print(
468+
f"boto3 delete_objects: {t_boto:.4f}s for {iterations} ops ({iterations / t_boto:.0f} ops/s)"
469+
)
470+
print(
471+
f"signurlarity delete_objects: {t_custom:.4f}s for {iterations} ops ({iterations / t_custom:.0f} ops/s)"
472+
)
473+
if t_custom > 0:
474+
speedup = t_boto / t_custom
475+
print(f"relative speed (signurlarity vs boto3): {speedup:.2f}x")
476+
if speedup > 1:
477+
print(f"✓ Signurlarity implementation is {speedup:.2f}x FASTER!")
478+
else:
479+
print(f"boto3 is {1 / speedup:.2f}x faster")
480+
481+
result_file.write_text(json.dumps(results, indent=2))
482+
483+
print("=" * 60)

0 commit comments

Comments
 (0)