diff --git a/awscli/clidriver.py b/awscli/clidriver.py index 1945dcbb9141..1a798aa2c5c0 100644 --- a/awscli/clidriver.py +++ b/awscli/clidriver.py @@ -41,6 +41,7 @@ ) from awscli.commands import CLICommand from awscli.compat import get_stderr_text_writer +from awscli.customizations.utils import uni_print from awscli.formatter import get_formatter from awscli.help import ( OperationHelpCommand, @@ -79,8 +80,6 @@ def main(): def create_clidriver(): session = botocore.session.Session(EnvironmentVariables) _set_user_agent_for_session(session) - # TODO check if full config plugins is empty or not. if it's not, we signal the warning for plugin support being provisional - # similarly, we check for api_versions config value here. load_plugins( session.full_config.get('plugins', {}), event_hooks=session.get_component('event_emitter'), @@ -547,6 +546,12 @@ def __call__(self, args, parsed_globals): parsed_args, self.arg_table ) + self._detect_binary_file_migration_change( + self._session, + parsed_args, + parsed_globals, + self.arg_table + ) event = f'calling-command.{self._parent_name}.{self._name}' override = self._emit_first_non_none_response( event, @@ -663,6 +668,45 @@ def _create_operation_parser(self, arg_table): parser = ArgTableArgParser(arg_table) return parser + def _detect_binary_file_migration_change( + self, + session, + parsed_args, + parsed_globals, + arg_table + ): + if ( + session.get_scoped_config() + .get('cli_binary_format', None) == 'raw-in-base64-out' + ): + # if cli_binary_format is set to raw-in-base64-out, then v2 behavior will + # be the same as v1, so there is no breaking change in this case. + return + if parsed_globals.v2_debug: + parsed_args_to_check = { + arg: getattr(parsed_args, arg) + for arg in vars(parsed_args) if getattr(parsed_args, arg) + } + + arg_values_to_check = [ + arg.py_name for arg in arg_table.values() + if arg.py_name in parsed_args_to_check + and arg.argument_model.type_name == 'blob' + and parsed_args_to_check[arg.py_name].startswith('file://') + ] + if arg_values_to_check: + uni_print( + 'AWS CLI v2 MIGRATION WARNING: When specifying a blob-type ' + 'parameter starting with `file://`, AWS CLI v2 will assume ' + 'the content of the file is already base64-encoded. To ' + 'maintain v1 behavior after upgrading to v2, set the ' + '`cli_binary_format` configuration variable to ' + '`raw-in-base64-out`. See https://docs.aws.amazon.com/cli/' + 'latest/userguide/cliv2-migration-changes.html#' + 'cliv2-migration-binaryparam.\n', + out_file=sys.stderr + ) + class CLIOperationCaller: """Call an AWS operation and format the response.""" diff --git a/awscli/customizations/cliinputjson.py b/awscli/customizations/cliinputjson.py index 01864e750f1a..47d58c0c3569 100644 --- a/awscli/customizations/cliinputjson.py +++ b/awscli/customizations/cliinputjson.py @@ -70,6 +70,10 @@ def add_to_call_parameters(self, call_parameters, parsed_args, try: # Try to load the JSON string into a python dictionary input_data = json.loads(retrieved_json) + self._session.register( + f"get-cli-input-json-data", + lambda **inner_kwargs: input_data + ) except ValueError as e: raise ParamError( self.name, "Invalid JSON: %s\nJSON received: %s" diff --git a/awscli/customizations/cloudformation/deploy.py b/awscli/customizations/cloudformation/deploy.py index c7572dfb0b47..a734453e2e0f 100644 --- a/awscli/customizations/cloudformation/deploy.py +++ b/awscli/customizations/cloudformation/deploy.py @@ -24,6 +24,7 @@ from awscli.customizations.commands import BasicCommand from awscli.compat import get_stdout_text_writer +from awscli.customizations.utils import uni_print from awscli.utils import create_nested_client, write_exception LOG = logging.getLogger(__name__) @@ -321,12 +322,13 @@ def _run_main(self, parsed_args, parsed_globals): parsed_args.execute_changeset, parsed_args.role_arn, parsed_args.notification_arns, s3_uploader, tags, parsed_args.fail_on_empty_changeset, - parsed_args.disable_rollback) + parsed_args.disable_rollback, getattr(parsed_globals, 'v2_debug', False)) def deploy(self, deployer, stack_name, template_str, parameters, capabilities, execute_changeset, role_arn, notification_arns, s3_uploader, tags, - fail_on_empty_changeset=True, disable_rollback=False): + fail_on_empty_changeset=True, disable_rollback=False, + v2_debug=False): try: result = deployer.create_and_wait_for_changeset( stack_name=stack_name, @@ -339,10 +341,19 @@ def deploy(self, deployer, stack_name, template_str, tags=tags ) except exceptions.ChangeEmptyError as ex: - # TODO print the runtime check for cli v2 breakage. technically won't be breaking if --fail-on-empty-changeset is - # explicitly provided. but we cannot differentiate between whether fail-on-empty-changeset is true because it's default - # or because it's explicitly specified. if fail_on_empty_changeset: + if v2_debug: + uni_print( + 'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, ' + 'deploying an AWS CloudFormation Template that ' + 'results in an empty changeset will NOT result in an ' + 'error. You can add the -–no-fail-on-empty-changeset ' + 'flag to migrate to v2 behavior and resolve this ' + 'warning. See https://docs.aws.amazon.com/cli/latest/' + 'userguide/cliv2-migration-changes.html' + '#cliv2-migration-cfn.\n', + out_file=sys.stderr + ) raise write_exception(ex, outfile=get_stdout_text_writer()) return 0 diff --git a/awscli/customizations/globalargs.py b/awscli/customizations/globalargs.py index 2ef6b9ef6a15..96b599ce37ea 100644 --- a/awscli/customizations/globalargs.py +++ b/awscli/customizations/globalargs.py @@ -96,19 +96,129 @@ def resolve_cli_connect_timeout(parsed_args, session, **kwargs): _resolve_timeout(session, parsed_args, arg_name) def detect_migration_breakage(parsed_args, remaining_args, session, **kwargs): - if parsed_args.v2_debug: - url_params = [param for param in remaining_args if param.startswith('http://') or param.startswith('https://')] - if parsed_args.command == 'ecr' and remaining_args[0] == 'get-login': - uni_print('AWS CLI v2 MIGRATION WARNING: The ecr get-login command has been removed in AWS CLI v2. See https://docs.aws.amazon.com/cli/latest/userguide/cliv2-migration-changes.html#cliv2-migration-ecr-get-login.\n') - if url_params and session.full_config.get('cli_follow_urlparam', True): - uni_print('AWS CLI v2 MIGRATION WARNING: For input parameters that have a prefix of http:// or https://, AWS CLI v2 will no longer automatically request the content of the URL for the parameter, and the cli_follow_urlparam option has been removed. See https://docs.aws.amazon.com/cli/latest/userguide/cliv2-migration-changes.html#cliv2-migration-paramfile.\n') - for working, obsolete in HIDDEN_ALIASES.items(): - working_split = working.split('.') - working_service = working_split[0] - working_cmd = working_split[1] - working_param = working_split[2] - if parsed_args.command == working_service and remaining_args[0] == working_cmd and f"--{working_param}" in remaining_args: - uni_print('AWS CLI v2 MIGRATION WARNING: You have entered command arguments that uses at least 1 of 21 hidden aliases that were removed in AWS CLI v2. See https://docs.aws.amazon.com/cli/latest/userguide/cliv2-migration-changes.html#cliv2-migration-aliases.\n') + if not parsed_args.v2_debug: + return + url_params = [ + param for param in remaining_args + if param.startswith('http://') or param.startswith('https://') + ] + region = parsed_args.region or session.get_config_variable('region') + s3_config = session.get_config_variable('s3') + if 'PYTHONUTF8' in os.environ or 'PYTHONIOENCODING' in os.environ: + if 'AWS_CLI_FILE_ENCODING' not in os.environ: + uni_print( + 'AWS CLI v2 MIGRATION WARNING: The PYTHONUTF8 and ' + 'PYTHONIOENCODING environment variables are unsupported ' + 'in AWS CLI v2. AWS CLI v2 uses AWS_CLI_FILE_ENCODING ' + 'instead, set this environment variable to resolve this. ' + 'See https://docs.aws.amazon.com/cli/latest/userguide/' + 'cliv2-migration-changes.html' + '#cliv2-migration-encodingenvvar.\n', + out_file=sys.stderr + ) + if ( + s3_config is not None and s3_config + .get( + 'us_east_1_regional_endpoint', + 'legacy' + ) == 'legacy' and region in ('us-east-1', None) + ): + session.register( + 'request-created.s3.*', + warn_if_east_configured_global_endpoint + ) + if session.get_config_variable('api_versions'): + uni_print( + 'AWS CLI v2 MIGRATION WARNING: The AWS CLI v2 does not support ' + 'calling earlier versions of AWS service APIs via the ' + '`api_versions` configuration file setting. To migrate to v2 ' + 'behavior and resolve this warning, remove the `api_versions` ' + 'setting in the configuration file. See ' + 'https://docs.aws.amazon.com/cli/latest/userguide/' + 'cliv2-migration-changes.html#cliv2-migration-api-versions.\n', + out_file = sys.stderr + ) + if session.full_config.get('plugins', {}): + uni_print( + 'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, plugin support ' + 'is provisional. If you rely on plugins, be sure to lock into ' + 'a particular version of the AWS CLI and test the ' + 'functionality of your plugins for each upgrade. See ' + 'https://docs.aws.amazon.com/cli/latest/userguide/' + 'cliv2-migration-changes.html#' + 'cliv2-migration-profile-plugins\n', + out_file=sys.stderr + ) + if parsed_args.command == 'ecr' and remaining_args[0] == 'get-login': + uni_print( + 'AWS CLI v2 MIGRATION WARNING: The ecr get-login command has ' + 'been removed in AWS CLI v2. See https://docs.aws.amazon.com/' + 'cli/latest/userguide/cliv2-migration-changes.html' + '#cliv2-migration-ecr-get-login.\n', + out_file=sys.stderr + ) + if url_params and session.full_config.get('cli_follow_urlparam', True): + uni_print( + 'AWS CLI v2 MIGRATION WARNING: For input parameters that have ' + 'a prefix of http:// or https://, AWS CLI v2 will no longer ' + 'automatically request the content of the URL for the ' + 'parameter, and the cli_follow_urlparam option has been ' + 'removed. See https://docs.aws.amazon.com/cli/latest/' + 'userguide/cliv2-migration-changes.html' + '#cliv2-migration-paramfile.\n', + out_file=sys.stderr + ) + for working, obsolete in HIDDEN_ALIASES.items(): + working_split = working.split('.') + working_service = working_split[0] + working_cmd = working_split[1] + working_param = working_split[2] + if ( + parsed_args.command == working_service + and remaining_args[0] == working_cmd + and f"--{working_param}" in remaining_args + ): + uni_print( + 'AWS CLI v2 MIGRATION WARNING: You have entered command ' + 'arguments that uses at least 1 of 21 hidden aliases that ' + 'were removed in AWS CLI v2. See ' + 'https://docs.aws.amazon.com/cli/latest/userguide' + '/cliv2-migration-changes.html#cliv2-migration-aliases.\n', + out_file=sys.stderr + ) + session.register('choose-signer.s3.*', warn_if_sigv2) + +def warn_if_east_configured_global_endpoint(request, operation_name, **kwargs): + # The regional us-east-1 endpoint is used in certain cases (e.g. + # FIPS/Dual-Stack is enabled). Rather than duplicating this logic + # from botocore, we check the endpoint URL directly. + if 's3.amazonaws.com' in request.url: + uni_print( + 'AWS CLI v2 MIGRATION WARNING: When you configure AWS CLI v2 ' + 'to use the `us-east-1` region, it uses the true regional ' + 'endpoint rather than the global endpoint. To continue using ' + 'the global endpoint on v2, configure the region to be ' + '`aws-global`. See https://docs.aws.amazon.com/cli/latest/' + 'userguide/cliv2-migration-changes.html' + '#cliv2-migration-s3-regional-endpoint.\n', + out_file=sys.stderr + ) + +def warn_if_sigv2( + signing_name, + region_name, + signature_version, + context, + **kwargs +): + if context.get('auth_type', None) == 'v2': + uni_print( + 'AWS CLI v2 MIGRATION WARNING: The AWS CLI v2 only uses Signature ' + 'v4 to authenticate Amazon S3 requests. Run the command `aws ' + 'configure set s3.signature_version s3v4` to migrate to v4 and ' + 'resolve this.\n', + out_file=sys.stderr + ) def resolve_cli_read_timeout(parsed_args, session, **kwargs): arg_name = 'read_timeout' diff --git a/awscli/customizations/paginate.py b/awscli/customizations/paginate.py index fe1f3f140112..836b338b0fef 100644 --- a/awscli/customizations/paginate.py +++ b/awscli/customizations/paginate.py @@ -135,6 +135,9 @@ def unify_paging_params(argument_table, operation_model, event_name, _remove_existing_paging_arguments(argument_table, paginator_config) parsed_args_event = event_name.replace('building-argument-table.', 'operation-args-parsed.') + call_parameters_event = event_name.replace( + 'building-argument-table', 'calling-command' + ) shadowed_args = {} add_paging_argument(argument_table, 'starting-token', PageArgument('starting-token', STARTING_TOKEN_HELP, @@ -168,6 +171,14 @@ def unify_paging_params(argument_table, operation_model, event_name, partial(check_should_enable_pagination, list(_get_all_cli_input_tokens(paginator_config)), shadowed_args, argument_table)) + session.register( + call_parameters_event, + partial( + check_should_enable_pagination_call_parameters, + session, + list(_get_all_input_tokens(paginator_config)), + ), + ) def add_paging_argument(argument_table, arg_name, argument, shadowed_args): @@ -240,6 +251,18 @@ def _get_all_cli_input_tokens(pagination_config): yield cli_name +# Get all tokens but return them in API namespace rather than CLI namespace +def _get_all_input_tokens(pagination_config): + # Get all input tokens including the limit_key + # if it exists. + tokens = _get_input_tokens(pagination_config) + for token_name in tokens: + yield token_name + if 'limit_key' in pagination_config: + key_name = pagination_config['limit_key'] + yield key_name + + def _get_input_tokens(pagination_config): tokens = pagination_config['input_token'] if not isinstance(tokens, list): @@ -253,6 +276,45 @@ def _get_cli_name(param_objects, token_name): return param.cli_name.lstrip('-') +def check_should_enable_pagination_call_parameters( + session, + input_tokens, + call_parameters, + parsed_args, + parsed_globals, + **kwargs +): + """ + Check for pagination args in the actual calling arguments passed to + the function. + + If the user is using the --cli-input-json parameter to provide JSON + parameters they are all in the API naming space rather than the CLI + naming space and would be missed by the processing above. This function + gets called on the calling-command event. + """ + if parsed_globals.v2_debug: + cli_input_json_data = session.emit_first_non_none_response( + f"get-cli-input-json-data", + ) + if cli_input_json_data is None: + cli_input_json_data = {} + pagination_params_in_input_tokens = [ + param for param in cli_input_json_data if param in input_tokens + ] + if pagination_params_in_input_tokens: + uni_print( + 'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, if you specify ' + 'pagination parameters by using a file with the ' + '`--cli-input-json` parameter, automatic pagination will be ' + 'turned off. This is not the case in v1. See ' + 'https://docs.aws.amazon.com/cli/latest/userguide/' + 'cliv2-migration-changes.html' + '#cliv2-migration-skeleton-paging.\n', + out_file=sys.stderr + ) + + class PageArgument(BaseCLIArgument): type_map = { 'string': str, diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index 3f3a2834c6d5..665fa9ae2603 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -767,6 +767,7 @@ def _run_main(self, parsed_args, parsed_globals): cmd_params.add_verify_ssl(parsed_globals) cmd_params.add_page_size(parsed_args) cmd_params.add_paths(parsed_args.paths) + cmd_params.add_v2_debug(parsed_globals) runtime_config = transferconfig.RuntimeConfig().build_config( **self._session.get_scoped_config().get('s3', {})) @@ -1056,6 +1057,21 @@ def run(self): result_queue = queue.Queue() operation_name = cmd_translation[paths_type] + if self.parameters['v2_debug']: + if operation_name == 'copy': + uni_print( + 'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, object ' + 'properties will be copied from the source in multipart ' + 'copies between S3 buckets. This may result in extra S3 ' + 'API calls being made. Breakage may occur if the principal ' + 'does not have permission to call these extra APIs. This ' + 'warning cannot be resolved. See ' + 'https://docs.aws.amazon.com/cli/latest/userguide/' + 'cliv2-migration-changes.html' + '#cliv2-migration-s3-copy-metadata\n\n', + out_file=sys.stderr + ) + fgen_kwargs = { 'client': self._source_client, 'operation_name': operation_name, 'follow_symlinks': self.parameters['follow_symlinks'], @@ -1447,6 +1463,9 @@ def add_verify_ssl(self, parsed_globals): def add_page_size(self, parsed_args): self.parameters['page_size'] = getattr(parsed_args, 'page_size', None) + def add_v2_debug(self, parsed_globals): + self.parameters['v2_debug'] = parsed_globals.v2_debug + def _validate_sse_c_args(self): self._validate_sse_c_arg() self._validate_sse_c_arg('sse_c_copy_source') diff --git a/awscli/customizations/scalarparse.py b/awscli/customizations/scalarparse.py index a080c9082146..7589897fe665 100644 --- a/awscli/customizations/scalarparse.py +++ b/awscli/customizations/scalarparse.py @@ -27,9 +27,13 @@ in the future. """ +import sys + from botocore.utils import parse_timestamp from botocore.exceptions import ProfileNotFound +from awscli.customizations.utils import uni_print + def register_scalar_parser(event_handlers): event_handlers.register_first( @@ -44,7 +48,7 @@ def iso_format(value): return parse_timestamp(value).isoformat() -def add_timestamp_parser(session): +def add_timestamp_parser(session, v2_debug): factory = session.get_component('response_parser_factory') try: timestamp_format = session.get_scoped_config().get( @@ -64,8 +68,26 @@ def add_timestamp_parser(session): # parser (which parses to a datetime.datetime object) with the # identity function which prints the date exactly the same as it comes # across the wire. - # TODO create an inner function that wraps identity here. it'll signal to print the runtime check. - timestamp_parser = identity + encountered_timestamp = False + def identity_with_warning(x): + # To prevent printing the same warning for each timestamp in the + # response, we utilize a reference to a nonlocal variable to track + # if we have already printed the warning. + nonlocal encountered_timestamp + if not encountered_timestamp: + encountered_timestamp = True + uni_print( + 'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, all timestamp ' + 'response values are returned in the ISO 8601 format. To ' + 'migrate to v2 behavior and resolve this warning, set the ' + 'configuration variable `cli_timestamp_format` to `iso8601`. ' + 'See https://docs.aws.amazon.com/cli/latest/userguide/' + 'cliv2-migration-changes.html#cliv2-migration-timestamp.\n', + out_file=sys.stderr + ) + return identity(x) + + timestamp_parser = identity_with_warning if v2_debug else identity elif timestamp_format == 'iso8601': timestamp_parser = iso_format else: @@ -74,7 +96,7 @@ def add_timestamp_parser(session): factory.set_parser_defaults(timestamp_parser=timestamp_parser) -def add_scalar_parsers(session, **kwargs): +def add_scalar_parsers(session, parsed_args, **kwargs): factory = session.get_component('response_parser_factory') factory.set_parser_defaults(blob_parser=identity) - add_timestamp_parser(session) + add_timestamp_parser(session, parsed_args.v2_debug) diff --git a/tests/functional/test_api_versions.py b/tests/functional/test_api_versions.py index c30da1b6a56a..2c3d339ab1ba 100644 --- a/tests/functional/test_api_versions.py +++ b/tests/functional/test_api_versions.py @@ -10,7 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -from awscli.testutils import create_clidriver +from awscli.testutils import create_clidriver, capture_output from awscli.testutils import BaseAWSCommandParamsTest, FileCreator @@ -49,3 +49,19 @@ def test_command_interface_reflects_api_version(self): cmdline = 'ec2 describe-nat-gateways' _, stderr, _ = self.run_cmd(cmdline, expected_rc=2) self.assertIn("Invalid choice: 'describe-nat-gateways'", stderr) + + def test_v2_debug_migration_warning(self): + cmdline = 'ec2 describe-instances --v2-debug' + _, stderr, _ = self.run_cmd(cmdline) + # Make sure that the correct api version is used for the client + # by checking the version that was sent in the request. + self.assertEqual(self.last_params['Version'], self.api_version) + # Make sure that the migration warning is printed since the user + # specified --v2-debug + self.assertIn( + 'AWS CLI v2 MIGRATION WARNING: The AWS CLI v2 does not support ' + 'calling earlier versions of AWS service APIs via the ' + '`api_versions` configuration file setting.', + stderr + ) + diff --git a/tests/unit/customizations/cloudformation/test_deploy.py b/tests/unit/customizations/cloudformation/test_deploy.py index 8aac8ae3be4f..894bb3dc9aa5 100644 --- a/tests/unit/customizations/cloudformation/test_deploy.py +++ b/tests/unit/customizations/cloudformation/test_deploy.py @@ -120,7 +120,8 @@ def test_command_invoked(self, mock_yaml_parse): None, fake_tags, True, - True + True, + False ) self.deploy_command.parse_key_value_arg.assert_has_calls([ @@ -207,7 +208,8 @@ def test_s3_uploader_is_configured_properly(self, s3UploaderMock, s3UploaderObject, [{"Key": "tagkey1", "Value": "tagvalue1"}], True, - True + True, + False ) s3UploaderMock.assert_called_once_with(mock.ANY, diff --git a/tests/unit/customizations/s3/test_subcommands.py b/tests/unit/customizations/s3/test_subcommands.py index 09dd0a3db667..c0b5af9eb804 100644 --- a/tests/unit/customizations/s3/test_subcommands.py +++ b/tests/unit/customizations/s3/test_subcommands.py @@ -450,7 +450,8 @@ def test_run_cp_put(self): 'paths_type': 'locals3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, - 'is_stream': False, 'source_region': None, 'metadata': None} + 'is_stream': False, 'source_region': None, 'metadata': None, + 'v2_debug': False} config = RuntimeConfig().build_config() cmd_arc = CommandArchitecture(self.session, 'cp', params, config) cmd_arc.set_clients() @@ -470,7 +471,8 @@ def test_error_on_same_line_as_status(self): 'paths_type': 'locals3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, - 'is_stream': False, 'source_region': None, 'metadata': None} + 'is_stream': False, 'source_region': None, 'metadata': None, + 'v2_debug': False} self.http_response.status_code = 400 self.parsed_responses = [{'Error': { 'Code': 'BucketNotExists', @@ -501,7 +503,7 @@ def test_run_cp_get(self): 'paths_type': 's3local', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, - 'is_stream': False, 'source_region': None} + 'is_stream': False, 'source_region': None, 'v2_debug': False} self.parsed_responses = [{"ETag": "abcd", "ContentLength": 100, "LastModified": "2014-01-09T20:45:49.000Z"}] config = RuntimeConfig().build_config() @@ -524,7 +526,7 @@ def test_run_cp_copy(self): 'paths_type': 's3s3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, - 'is_stream': False, 'source_region': None} + 'is_stream': False, 'source_region': None, 'v2_debug': False} self.parsed_responses = [{"ETag": "abcd", "ContentLength": 100, "LastModified": "2014-01-09T20:45:49.000Z"}] config = RuntimeConfig().build_config() @@ -548,7 +550,7 @@ def test_run_mv(self): 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, 'is_stream': False, 'source_region': None, - 'is_move': True} + 'is_move': True, 'v2_debug': False} self.parsed_responses = [{"ETag": "abcd", "ContentLength": 100, "LastModified": "2014-01-09T20:45:49.000Z"}] config = RuntimeConfig().build_config() @@ -571,7 +573,8 @@ def test_run_remove(self): 'paths_type': 's3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, - 'is_stream': False, 'source_region': None} + 'is_stream': False, 'source_region': None, + 'v2_debug': False} self.parsed_responses = [{"ETag": "abcd", "ContentLength": 100, "LastModified": "2014-01-09T20:45:49.000Z"}] config = RuntimeConfig().build_config() @@ -598,7 +601,8 @@ def test_run_sync(self): 'paths_type': 'locals3', 'region': 'us-east-1', 'endpoint_url': None, 'verify_ssl': None, 'follow_symlinks': True, 'page_size': None, - 'is_stream': False, 'source_region': 'us-west-2'} + 'is_stream': False, 'source_region': 'us-west-2', + 'v2_debug': False} self.parsed_responses = [ {"CommonPrefixes": [], "Contents": [ {"Key": "text1.txt", "Size": 100, @@ -613,6 +617,31 @@ def test_run_sync(self): output_str = "(dryrun) upload: %s to %s" % (rel_local_file, s3_file) self.assertIn(output_str, self.output.getvalue()) + def test_v2_debug_mv(self): + s3_file = 's3://' + self.bucket + '/' + 'text1.txt' + filters = [['--include', '*']] + params = {'dir_op': False, 'quiet': False, 'dryrun': True, + 'src': s3_file, 'dest': s3_file, 'filters': filters, + 'paths_type': 's3s3', 'region': 'us-east-1', + 'endpoint_url': None, 'verify_ssl': None, + 'follow_symlinks': True, 'page_size': None, + 'is_stream': False, 'source_region': None, + 'is_move': True, 'v2_debug': True} + self.parsed_responses = [{"ETag": "abcd", "ContentLength": 100, + "LastModified": "2014-01-09T20:45:49.000Z"}] + config = RuntimeConfig().build_config() + cmd_arc = CommandArchitecture(self.session, 'mv', params, config) + cmd_arc.set_clients() + cmd_arc.create_instructions() + self.patch_make_request() + cmd_arc.run() + warning_str = 'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, object '\ + 'properties will be copied from the source in multipart '\ + 'copies between S3 buckets.' + output_str = f"(dryrun) move: {s3_file} to {s3_file}" + self.assertIn(warning_str, self.err_output.getvalue()) + self.assertIn(output_str, self.output.getvalue()) + class CommandParametersTest(unittest.TestCase): def setUp(self): diff --git a/tests/unit/customizations/test_globalargs.py b/tests/unit/customizations/test_globalargs.py index 8316b7c9229b..f3bf17b4a680 100644 --- a/tests/unit/customizations/test_globalargs.py +++ b/tests/unit/customizations/test_globalargs.py @@ -14,7 +14,7 @@ from botocore import UNSIGNED import os -from awscli.testutils import mock, unittest +from awscli.testutils import mock, unittest, capture_output from awscli.customizations import globalargs @@ -185,3 +185,63 @@ def test_cli_connect_timeout_for_blocking(self): self.assertEqual(parsed_args.connect_timeout, None) self.assertEqual( session.get_default_client_config().connect_timeout, None) + + def test_v2_debug_python_utf8_env_var(self): + parsed_args = FakeParsedArgs(v2_debug=True) + session = get_session() + environ = {'PYTHONUTF8': '1'} + with mock.patch('os.environ', environ): + with capture_output() as output: + globalargs.detect_migration_breakage(parsed_args, [], session) + self.assertIn( + 'AWS CLI v2 MIGRATION WARNING: The PYTHONUTF8 and ' + 'PYTHONIOENCODING environment variables are unsupported ' + 'in AWS CLI v2.', + output.stderr.getvalue() + ) + + def test_v2_debug_python_utf8_resolved_env_var(self): + parsed_args = FakeParsedArgs(v2_debug=True) + session = get_session() + environ = {'PYTHONUTF8': '1', 'AWS_CLI_FILE_ENCODING': 'UTF-8'} + with mock.patch('os.environ', environ): + with capture_output() as output: + globalargs.detect_migration_breakage(parsed_args, [], session) + self.assertNotIn( + 'AWS CLI v2 MIGRATION WARNING: The PYTHONUTF8 and ' + 'PYTHONIOENCODING environment variables are unsupported ' + 'in AWS CLI v2.', + output.stderr.getvalue() + ) + + def test_v2_debug_python_io_encoding_env_var(self): + parsed_args = FakeParsedArgs(v2_debug=True) + session = get_session() + environ = {'PYTHONIOENCODING': 'UTF8'} + with mock.patch('os.environ', environ): + with capture_output() as output: + globalargs.detect_migration_breakage(parsed_args, [], session) + self.assertIn( + 'AWS CLI v2 MIGRATION WARNING: The PYTHONUTF8 and ' + 'PYTHONIOENCODING environment variables are unsupported ' + 'in AWS CLI v2.', + output.stderr.getvalue() + ) + + def test_v2_debug_s3_sigv2(self): + parsed_args = FakeParsedArgs(v2_debug=True) + session = get_session() + globalargs.detect_migration_breakage(parsed_args, [], session) + with capture_output() as output: + session.emit( + 'choose-signer.s3.*', + signing_name='s3', + region_name='us-west-2', + signature_version='v2', + context={'auth_type': 'v2'}, + ) + self.assertIn( + 'AWS CLI v2 MIGRATION WARNING: The AWS CLI v2 only uses Signature ' + 'v4 to authenticate Amazon S3 requests.', + output.stderr.getvalue() + ) diff --git a/tests/unit/customizations/test_paginate.py b/tests/unit/customizations/test_paginate.py index cb362a0fc631..03eed8e45f30 100644 --- a/tests/unit/customizations/test_paginate.py +++ b/tests/unit/customizations/test_paginate.py @@ -10,10 +10,12 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +from functools import partial + import pytest from awscli.customizations.paginate import PageArgument -from awscli.testutils import mock, unittest +from awscli.testutils import mock, unittest, capture_output from botocore.exceptions import DataNotFoundError, PaginationError from botocore.model import OperationModel @@ -215,6 +217,7 @@ def setUp(self): self.parsed_args.starting_token = None self.parsed_args.page_size = None self.parsed_args.max_items = None + self.call_parameters = {} def test_should_not_enable_pagination(self): # Here the user has specified a manual pagination argument, @@ -307,6 +310,91 @@ def test_shadowed_args_are_replaced_when_pagination_set_off(self): self.assertEqual(arg_table['foo'], mock.sentinel.ORIGINAL_ARG) +class TestPaginateV2Debug(TestPaginateBase): + def setUp(self): + super().setUp() + self.parsed_globals = mock.Mock() + self.parsed_args = mock.Mock() + self.parsed_args.starting_token = None + self.parsed_args.page_size = None + self.parsed_args.max_items = None + self.call_parameters = {} + + def _mock_emit_first_non_none_response( + self, + mock_input_json_data, + event_name + ): + if event_name == 'get-cli-input-json-data': + return mock_input_json_data + return None + + def test_v2_debug_call_parameters(self): + # Here the user has specified a manual pagination argument, + # via CLI Input JSON and specified v2-debug, so a + # migration warning should be printed. + # From setUp(), the limit_key is 'Bar' + input_tokens = ['Foo', 'Bar'] + self.parsed_globals.v2_debug = True + self.parsed_globals.paginate = True + self.session.emit_first_non_none_response.side_effect = partial( + self._mock_emit_first_non_none_response, + {'Bar': 10} + ) + # Corresponds to --bar 10 + self.call_parameters['Foo'] = None + self.call_parameters['Bar'] = 10 + with capture_output() as output: + paginate.check_should_enable_pagination_call_parameters( + self.session, + input_tokens, + self.call_parameters, + {}, + self.parsed_globals + ) + # We should have printed the migration warning + # because the user specified {Bar: 10} in the input JSON + self.assertIn( + 'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, if you specify ' + 'pagination parameters by using a file with the ' + '`--cli-input-json` parameter, automatic pagination will be ' + 'turned off.', + output.stderr.getvalue() + ) + + def test_v2_debug_call_params_does_not_print_for_cmd_args(self): + # Here the user has specified a pagination argument as a command + # argument and specified v2-debug, so the migration warning should NOT + # be printed. From setUp(), the limit_key is 'Bar' + input_tokens = ['Foo', 'Bar'] + self.parsed_globals.v2_debug = True + self.parsed_globals.paginate = True + self.session.emit_first_non_none_response.side_effect = partial( + self._mock_emit_first_non_none_response, + None + ) + # Corresponds to --bar 10 + self.call_parameters['Foo'] = None + self.call_parameters['Bar'] = 10 + with capture_output() as output: + paginate.check_should_enable_pagination_call_parameters( + self.session, + input_tokens, + self.call_parameters, + {}, + self.parsed_globals + ) + # We should not have printed the warning because + # the user did not specify any params through CLI input JSON + self.assertNotIn( + 'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, if you specify ' + 'pagination parameters by using a file with the ' + '`--cli-input-json` parameter, automatic pagination will be ' + 'turned off.', + output.stderr.getvalue() + ) + + class TestEnsurePagingParamsNotSet(TestPaginateBase): def setUp(self): super(TestEnsurePagingParamsNotSet, self).setUp() diff --git a/tests/unit/customizations/test_scalarparse.py b/tests/unit/customizations/test_scalarparse.py index 0dd0afb53d37..f04588396e3c 100644 --- a/tests/unit/customizations/test_scalarparse.py +++ b/tests/unit/customizations/test_scalarparse.py @@ -32,7 +32,7 @@ def test_scalar_parsers_set(self): session = mock.Mock() session.get_scoped_config.return_value = {'cli_timestamp_format': 'none'} - scalarparse.add_scalar_parsers(session) + scalarparse.add_scalar_parsers(session, mock.Mock(v2_debug=False)) session.get_component.assert_called_with('response_parser_factory') factory = session.get_component.return_value expected = [mock.call(blob_parser=scalarparse.identity), @@ -45,7 +45,7 @@ def test_choose_none_timestamp_formatter(self): session.get_scoped_config.return_value = {'cli_timestamp_format': 'none'} factory = session.get_component.return_value - scalarparse.add_scalar_parsers(session) + scalarparse.add_scalar_parsers(session, mock.Mock(v2_debug=False)) factory.set_parser_defaults.assert_called_with( timestamp_parser=scalarparse.identity) @@ -54,7 +54,7 @@ def test_choose_iso_timestamp_formatter(self): session.get_scoped_config.return_value = {'cli_timestamp_format': 'iso8601'} factory = session.get_component.return_value - scalarparse.add_scalar_parsers(session) + scalarparse.add_scalar_parsers(session, mock.Mock(v2_debug=False)) factory.set_parser_defaults.assert_called_with( timestamp_parser=scalarparse.iso_format) @@ -64,12 +64,12 @@ def test_choose_invalid_timestamp_formatter(self): 'foobar'} session.get_component.return_value with self.assertRaises(ValueError): - scalarparse.add_scalar_parsers(session) + scalarparse.add_scalar_parsers(session, mock.Mock(v2_debug=False)) def test_choose_timestamp_parser_profile_not_found(self): session = mock.Mock(spec=Session) session.get_scoped_config.side_effect = ProfileNotFound(profile='foo') factory = session.get_component.return_value - scalarparse.add_scalar_parsers(session) + scalarparse.add_scalar_parsers(session, mock.Mock(v2_debug=False)) factory.set_parser_defaults.assert_called_with( timestamp_parser=scalarparse.identity) diff --git a/tests/unit/test_clidriver.py b/tests/unit/test_clidriver.py index 1b5e2dfa68ec..a88e13ed76ed 100644 --- a/tests/unit/test_clidriver.py +++ b/tests/unit/test_clidriver.py @@ -104,7 +104,8 @@ GET_VARIABLE = { 'provider': 'aws', 'output': 'json', - 'api_versions': {} + 'api_versions': {}, + 'cli_binary_format': 'raw-in-base64-out', } @@ -236,6 +237,9 @@ def get_config_variable(self, name): return GET_VARIABLE[name] return self.session_vars[name] + def get_scoped_config(self): + return GET_VARIABLE + def get_service_model(self, name, api_version=None): return botocore.model.ServiceModel( MINI_SERVICE, service_name='s3')