Skip to content

Commit 08d5ba6

Browse files
Copilotpvliesdonk
andcommitted
fix: restructure export_tar to properly stream without blocking
- Use async_docker_call to wrap exec_run call instead of asyncio.to_thread - Implement proper async streaming using async generator - Process chunks individually without loading entire stream into memory - Fix unused variable linting error - All 229 unit tests passing Co-authored-by: pvliesdonk <22190282+pvliesdonk@users.noreply.github.com>
1 parent 98b5934 commit 08d5ba6

File tree

1 file changed

+25
-18
lines changed

1 file changed

+25
-18
lines changed

src/mcp_devbench/managers/filesystem_manager.py

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -923,28 +923,35 @@ async def export_tar(
923923
tar_flags = "czf" if compress else "cf"
924924
full_cmd = f"tar -{tar_flags} - -C {shlex.quote(normalized_path)} ."
925925

926-
# Execute tar command and stream output - wrap in thread pool
927-
# Note: We need to handle streaming properly
928-
def run_tar_stream():
929-
return container.exec_run(
930-
["sh", "-c", full_cmd],
931-
user="1000:1000",
932-
stream=True,
933-
demux=False,
934-
)
935-
936-
exec_result = await asyncio.to_thread(run_tar_stream)
926+
# Execute tar command with streaming - wrap exec_run in thread pool
927+
exec_result = await async_docker_call(
928+
container.exec_run,
929+
["sh", "-c", full_cmd],
930+
user="1000:1000",
931+
stream=True,
932+
demux=False,
933+
)
937934

938-
# Stream the output in chunks (Docker handles chunking)
939-
# Wrap the iteration in a thread to avoid blocking
940-
def iter_chunks():
941-
for chunk in exec_result.output:
935+
# Stream the output in chunks without loading all into memory
936+
# The iteration over exec_result.output is blocking, so wrap it
937+
async def stream_chunks():
938+
"""Async generator that yields chunks from the blocking iterator."""
939+
def get_next_chunk(iterator):
940+
"""Get the next chunk from iterator in a thread-safe way."""
941+
try:
942+
return next(iterator), False # (chunk, done)
943+
except StopIteration:
944+
return None, True # (None, done)
945+
946+
iterator = iter(exec_result.output)
947+
while True:
948+
chunk, done = await asyncio.to_thread(get_next_chunk, iterator)
949+
if done:
950+
break
942951
if chunk:
943952
yield chunk
944953

945-
# Stream chunks from the thread to avoid blocking the event loop
946-
loop = asyncio.get_running_loop()
947-
for chunk in await asyncio.to_thread(lambda: list(iter_chunks())):
954+
async for chunk in stream_chunks():
948955
yield chunk
949956

950957
logger.info(f"Exported tar from {normalized_path} in container {container_id}")

0 commit comments

Comments
 (0)