|
13 | 13 | _logger: logging.Logger = logging.getLogger(__name__) |
14 | 14 |
|
15 | 15 |
|
16 | | -def _copy_objects(batch: List[Tuple[str, str]], use_threads: bool, |
17 | | - boto3_session: boto3.Session, s3_additional_kwargs: Optional[Dict[str, str]]) -> None: |
| 16 | +def _copy_objects( |
| 17 | + batch: List[Tuple[str, str]], |
| 18 | + use_threads: bool, |
| 19 | + boto3_session: boto3.Session, |
| 20 | + s3_additional_kwargs: Optional[Dict[str, str]], |
| 21 | +) -> None: |
18 | 22 | _logger.debug("len(batch): %s", len(batch)) |
19 | 23 | client_s3: boto3.client = _utils.client(service_name="s3", session=boto3_session) |
20 | 24 | resource_s3: boto3.resource = _utils.resource(service_name="s3", session=boto3_session) |
@@ -115,8 +119,12 @@ def merge_datasets( |
115 | 119 | raise exceptions.InvalidArgumentValue(f"{mode} is a invalid mode option.") |
116 | 120 |
|
117 | 121 | new_objects: List[str] = copy_objects( |
118 | | - paths=paths, source_path=source_path, target_path=target_path, |
119 | | - use_threads=use_threads, boto3_session=session, s3_additional_kwargs=s3_additional_kwargs |
| 122 | + paths=paths, |
| 123 | + source_path=source_path, |
| 124 | + target_path=target_path, |
| 125 | + use_threads=use_threads, |
| 126 | + boto3_session=session, |
| 127 | + s3_additional_kwargs=s3_additional_kwargs, |
120 | 128 | ) |
121 | 129 | _logger.debug("len(new_objects): %s", len(new_objects)) |
122 | 130 | return new_objects |
@@ -202,7 +210,6 @@ def copy_objects( |
202 | 210 | batch.append((path, path_final)) |
203 | 211 | _logger.debug("len(new_objects): %s", len(new_objects)) |
204 | 212 | _copy_objects( |
205 | | - batch=batch, use_threads=use_threads, |
206 | | - boto3_session=session, s3_additional_kwargs=s3_additional_kwargs |
| 213 | + batch=batch, use_threads=use_threads, boto3_session=session, s3_additional_kwargs=s3_additional_kwargs |
207 | 214 | ) |
208 | 215 | return new_objects |
0 commit comments