diff --git a/awscli/customizations/s3/results.py b/awscli/customizations/s3/results.py index 024fa7b79f4c..0013baf9b3eb 100644 --- a/awscli/customizations/s3/results.py +++ b/awscli/customizations/s3/results.py @@ -57,6 +57,7 @@ def _create_new_result_cls(name, extra_fields=None, base_cls=BaseResult): FailureResult = _create_new_result_cls('FailureResult', ['exception']) DryRunResult = _create_new_result_cls('DryRunResult') +SkipFileResult = _create_new_result_cls('SkipFileResult') ErrorResult = namedtuple('ErrorResult', ['exception']) @@ -128,10 +129,17 @@ def _on_failure(self, future, e): self._result_queue.put(error_result_cls(exception=e)) elif self._is_precondition_failed(e): LOGGER.debug( - "Warning: Skipping file %s as it already exists on %s", + "warning: skipping file %s as it already exists on %s", self._src, self._dest, ) + self._result_queue.put( + SkipFileResult( + transfer_type=self._transfer_type, + src=self._src, + dest=self._dest, + ) + ) else: self._result_queue.put( FailureResult( @@ -167,6 +175,7 @@ def __init__(self): self.files_transferred = 0 self.files_failed = 0 self.files_warned = 0 + self.files_skipped = 0 self.errors = 0 self.expected_bytes_transferred = 0 self.expected_files_transferred = 0 @@ -184,6 +193,7 @@ def __init__(self): SuccessResult: self._record_success_result, FailureResult: self._record_failure_result, WarningResult: self._record_warning_result, + SkipFileResult: self._record_skipped_file_result, ErrorResult: self._record_error_result, CtrlCResult: self._record_error_result, FinalTotalSubmissionsResult: self._record_final_expected_files, @@ -299,6 +309,10 @@ def _record_failure_result(self, result, **kwargs): self.files_failed += 1 self.files_transferred += 1 + def _record_skipped_file_result(self, result, **kwargs): + self.files_skipped += 1 + self.files_transferred += 1 + def _record_warning_result(self, **kwargs): self.files_warned += 1 @@ -359,6 +373,7 @@ def __init__(self, result_recorder, out_file=None, error_file=None): SuccessResult: self._print_success, FailureResult: self._print_failure, WarningResult: self._print_warning, + SkipFileResult: self._print_noop, ErrorResult: self._print_error, CtrlCResult: self._print_ctrl_c, DryRunResult: self._print_dry_run, @@ -435,6 +450,7 @@ def _redisplay_progress(self): self._add_progress_if_needed() def _add_progress_if_needed(self): + LOGGER.debug("Remaining Progrss %s", self._has_remaining_progress()) if self._has_remaining_progress(): self._print_progress() @@ -446,7 +462,6 @@ def _print_progress(self, **kwargs): - self._result_recorder.files_transferred ) ) - # Create the display statement. if self._result_recorder.expected_bytes_transferred > 0: bytes_completed = human_readable_size( @@ -504,10 +519,15 @@ def _adjust_statement_padding(self, print_statement, ending_char='\n'): return print_statement + ending_char def _has_remaining_progress(self): + LOGGER.debug( + "Expected Files %s", + self._result_recorder.expected_totals_are_final(), + ) if not self._result_recorder.expected_totals_are_final(): return True actual = self._result_recorder.files_transferred expected = self._result_recorder.expected_files_transferred + LOGGER.debug("Actual %s Expected %s", actual, expected) return actual != expected def _print_to_out_file(self, statement): diff --git a/awscli/customizations/s3/s3handler.py b/awscli/customizations/s3/s3handler.py index 2b882a3a4de1..e136ec34c1c9 100644 --- a/awscli/customizations/s3/s3handler.py +++ b/awscli/customizations/s3/s3handler.py @@ -30,6 +30,7 @@ ResultPrinter, ResultProcessor, ResultRecorder, + SkipFileResult, SuccessResult, ) from awscli.customizations.s3.subscribers import ( @@ -462,10 +463,16 @@ def _warn_if_file_exists_with_no_overwrite(self, fileinfo): if not self._cli_params.get('no_overwrite'): return False fileout = self._get_fileout(fileinfo) + result_kwargs = { + 'transfer_type': 'download', + 'src': fileinfo.src, + 'dest': fileinfo.dest, + } if os.path.exists(fileout): LOGGER.debug( f"warning: skipping {fileinfo.src} -> {fileinfo.dest}, file exists at destination" ) + self._result_queue.put(SkipFileResult(**result_kwargs)) return True return False @@ -538,11 +545,17 @@ def _warn_if_zero_byte_file_exists_with_no_overwrite(self, fileinfo): bucket, key = find_bucket_key(fileinfo.dest) client = fileinfo.source_client + result_kwargs = { + 'transfer_type': 'copy', + 'src': fileinfo.src, + 'dest': fileinfo.dest, + } try: client.head_object(Bucket=bucket, Key=key) LOGGER.debug( f"warning: skipping {fileinfo.src} -> {fileinfo.dest}, file exists at destination" ) + self._result_queue.put(SkipFileResult(**result_kwargs)) return True except ClientError as e: if e.response['Error']['Code'] == '404':