|
27 | 27 |
|
28 | 28 | _logger = logging.getLogger(__name__) |
29 | 29 |
|
30 | | - |
31 | | -async def _run_cli_command( |
32 | | - command: str, |
33 | | - *, |
34 | | - output_handlers: list[Callable[[str], Awaitable[None]]] | None = None, |
35 | | -) -> None: |
36 | | - """ |
37 | | - Raises: |
38 | | - ArchiveError: when it fails to execute the command |
39 | | - """ |
40 | | - |
41 | | - process = await asyncio.create_subprocess_shell( |
42 | | - command, |
43 | | - stdin=asyncio.subprocess.PIPE, |
44 | | - stdout=asyncio.subprocess.PIPE, |
45 | | - stderr=asyncio.subprocess.STDOUT, |
46 | | - ) |
47 | | - |
48 | | - async def read_stream( |
49 | | - stream, chunk_size: NonNegativeInt = _DEFAULT_CHUNK_SIZE |
50 | | - ) -> str: |
51 | | - command_output = "" |
52 | | - |
53 | | - # Initialize buffer to store lookbehind window |
54 | | - lookbehind_buffer = "" |
55 | | - |
56 | | - undecodable_chunk: bytes | None = None |
57 | | - |
58 | | - while True: |
59 | | - read_chunk = await stream.read(chunk_size) |
60 | | - if not read_chunk: |
61 | | - # Process remaining buffer if any |
62 | | - if lookbehind_buffer and output_handlers: |
63 | | - await asyncio.gather( |
64 | | - *[handler(lookbehind_buffer) for handler in output_handlers] |
65 | | - ) |
66 | | - break |
67 | | - |
68 | | - try: |
69 | | - if undecodable_chunk: |
70 | | - chunk = (undecodable_chunk + read_chunk).decode("utf-8") |
71 | | - undecodable_chunk = None |
72 | | - else: |
73 | | - chunk = read_chunk.decode("utf-8") |
74 | | - except UnicodeDecodeError: |
75 | | - undecodable_chunk = read_chunk |
76 | | - continue |
77 | | - |
78 | | - command_output += chunk |
79 | | - |
80 | | - # Combine lookbehind buffer with new chunk |
81 | | - chunk_to_emit = lookbehind_buffer + chunk |
82 | | - |
83 | | - if output_handlers: |
84 | | - await asyncio.gather( |
85 | | - *[handler(chunk_to_emit) for handler in output_handlers] |
86 | | - ) |
87 | | - |
88 | | - # Keep last window_size characters for next iteration |
89 | | - lookbehind_buffer = chunk_to_emit[-chunk_size:] |
90 | | - |
91 | | - return command_output |
92 | | - |
93 | | - # Wait for the process to complete and all output to be processed |
94 | | - command_output, _ = await asyncio.gather( |
95 | | - asyncio.create_task(read_stream(process.stdout)), |
96 | | - process.wait(), |
97 | | - ) |
98 | | - |
99 | | - if process.returncode != os.EX_OK: |
100 | | - msg = f"Could not run '{command}' error: '{command_output}'" |
101 | | - raise ArchiveError(msg) |
102 | | - |
103 | | - |
104 | 30 | _TOTAL_BYTES_RE: Final[str] = r" (\d+)\s*bytes " |
105 | 31 | _FILE_COUNT_RE: Final[str] = r" (\d+)\s*files" |
106 | 32 | _PROGRESS_PERCENT_RE: Final[str] = r" (?:100|\d?\d)% " |
@@ -183,6 +109,85 @@ async def parse_chunk(self, chunk: str) -> None: |
183 | 109 | self.finished_emitted = True |
184 | 110 |
|
185 | 111 |
|
| 112 | +async def _output_reader( |
| 113 | + stream, |
| 114 | + *, |
| 115 | + output_handlers: list[Callable[[str], Awaitable[None]]] | None, |
| 116 | + chunk_size: NonNegativeInt = _DEFAULT_CHUNK_SIZE, |
| 117 | +) -> str: |
| 118 | + command_output = "" |
| 119 | + |
| 120 | + # Initialize buffer to store lookbehind window |
| 121 | + lookbehind_buffer = "" |
| 122 | + |
| 123 | + undecodable_chunk: bytes | None = None |
| 124 | + |
| 125 | + while True: |
| 126 | + read_chunk = await stream.read(chunk_size) |
| 127 | + if not read_chunk: |
| 128 | + # Process remaining buffer if any |
| 129 | + if lookbehind_buffer and output_handlers: |
| 130 | + await asyncio.gather( |
| 131 | + *[handler(lookbehind_buffer) for handler in output_handlers] |
| 132 | + ) |
| 133 | + break |
| 134 | + |
| 135 | + try: |
| 136 | + if undecodable_chunk: |
| 137 | + chunk = (undecodable_chunk + read_chunk).decode("utf-8") |
| 138 | + undecodable_chunk = None |
| 139 | + else: |
| 140 | + chunk = read_chunk.decode("utf-8") |
| 141 | + except UnicodeDecodeError: |
| 142 | + undecodable_chunk = read_chunk |
| 143 | + continue |
| 144 | + |
| 145 | + command_output += chunk |
| 146 | + |
| 147 | + # Combine lookbehind buffer with new chunk |
| 148 | + chunk_to_emit = lookbehind_buffer + chunk |
| 149 | + |
| 150 | + if output_handlers: |
| 151 | + await asyncio.gather( |
| 152 | + *[handler(chunk_to_emit) for handler in output_handlers] |
| 153 | + ) |
| 154 | + |
| 155 | + # Keep last window_size characters for next iteration |
| 156 | + lookbehind_buffer = chunk_to_emit[-chunk_size:] |
| 157 | + |
| 158 | + return command_output |
| 159 | + |
| 160 | + |
| 161 | +async def _run_cli_command( |
| 162 | + command: str, |
| 163 | + *, |
| 164 | + output_handlers: list[Callable[[str], Awaitable[None]]] | None = None, |
| 165 | +) -> None: |
| 166 | + """ |
| 167 | + Raises: |
| 168 | + ArchiveError: when it fails to execute the command |
| 169 | + """ |
| 170 | + |
| 171 | + process = await asyncio.create_subprocess_shell( |
| 172 | + command, |
| 173 | + stdin=asyncio.subprocess.PIPE, |
| 174 | + stdout=asyncio.subprocess.PIPE, |
| 175 | + stderr=asyncio.subprocess.STDOUT, |
| 176 | + ) |
| 177 | + |
| 178 | + # Wait for the process to complete and all output to be processed |
| 179 | + command_output, _ = await asyncio.gather( |
| 180 | + asyncio.create_task( |
| 181 | + _output_reader(process.stdout, output_handlers=output_handlers) |
| 182 | + ), |
| 183 | + process.wait(), |
| 184 | + ) |
| 185 | + |
| 186 | + if process.returncode != os.EX_OK: |
| 187 | + msg = f"Could not run '{command}' error: '{command_output}'" |
| 188 | + raise ArchiveError(msg) |
| 189 | + |
| 190 | + |
186 | 191 | async def archive_dir( |
187 | 192 | dir_to_compress: Path, |
188 | 193 | destination: Path, |
|
0 commit comments