Implement v2-debug checks for args, config, customizations, s3, and environment variables

awscli/clidriver.py

@@ -41,6 +41,7 @@
 )
 from awscli.commands import CLICommand
 from awscli.compat import get_stderr_text_writer
+from awscli.customizations.utils import uni_print
 from awscli.formatter import get_formatter
 from awscli.help import (
     OperationHelpCommand,
@@ -79,8 +80,6 @@ def main():
 def create_clidriver():
     session = botocore.session.Session(EnvironmentVariables)
     _set_user_agent_for_session(session)
-    # TODO check if full config plugins is empty or not. if it's not, we signal the warning for plugin support being provisional
-    # similarly, we check for api_versions config value here.
     load_plugins(
         session.full_config.get('plugins', {}),
         event_hooks=session.get_component('event_emitter'),
@@ -547,6 +546,12 @@ def __call__(self, args, parsed_globals):
             parsed_args, self.arg_table
         )
 
+        self._detect_binary_file_migration_change(
+            self._session,
+            parsed_args,
+            parsed_globals,
+            self.arg_table
+        )
         event = f'calling-command.{self._parent_name}.{self._name}'
         override = self._emit_first_non_none_response(
             event,
@@ -663,6 +668,45 @@ def _create_operation_parser(self, arg_table):
         parser = ArgTableArgParser(arg_table)
         return parser
 
+    def _detect_binary_file_migration_change(
+            self,
+            session,
+            parsed_args,
+            parsed_globals,
+            arg_table
+    ):
+        if (
+                session.get_scoped_config()
+                        .get('cli_binary_format', None) == 'raw-in-base64-out'
+        ):
+            # if cli_binary_format is set to raw-in-base64-out, then v2 behavior will
+            # be the same as v1, so there is no breaking change in this case.
+            return
+        if parsed_globals.v2_debug:
+            parsed_args_to_check = {
+                arg: getattr(parsed_args, arg)
+                for arg in vars(parsed_args) if getattr(parsed_args, arg)
+            }
+
+            arg_values_to_check = [
+                arg.py_name for arg in arg_table.values()
+                if arg.py_name in parsed_args_to_check
+                   and arg.argument_model.type_name == 'blob'
+                   and parsed_args_to_check[arg.py_name].startswith('file://')
+            ]
+            if arg_values_to_check:
+                uni_print(
+                    'AWS CLI v2 MIGRATION WARNING: When specifying a blob-type '
+                    'parameter starting with `file://`, AWS CLI v2 will assume '
+                    'the content of the file is already base64-encoded. To '
+                    'maintain v1 behavior after upgrading to v2, set the '
+                    '`cli_binary_format` configuration variable to '
+                    '`raw-in-base64-out`. See https://docs.aws.amazon.com/cli/'
+                    'latest/userguide/cliv2-migration-changes.html#'
+                    'cliv2-migration-binaryparam.\n',
+                    out_file=sys.stderr
+                )
+
 
 class CLIOperationCaller:
     """Call an AWS operation and format the response."""

awscli/customizations/cliinputjson.py

@@ -70,6 +70,10 @@ def add_to_call_parameters(self, call_parameters, parsed_args,
             try:
                 # Try to load the JSON string into a python dictionary
                 input_data = json.loads(retrieved_json)
+                self._session.register(
+                    f"get-cli-input-json-data",
+                    lambda **inner_kwargs: input_data
+                )
             except ValueError as e:
                 raise ParamError(
                     self.name, "Invalid JSON: %s\nJSON received: %s"

awscli/customizations/cloudformation/deploy.py

@@ -24,6 +24,7 @@
 
 from awscli.customizations.commands import BasicCommand
 from awscli.compat import get_stdout_text_writer
+from awscli.customizations.utils import uni_print
 from awscli.utils import create_nested_client, write_exception
 
 LOG = logging.getLogger(__name__)
@@ -321,12 +322,13 @@ def _run_main(self, parsed_args, parsed_globals):
                            parsed_args.execute_changeset, parsed_args.role_arn,
                            parsed_args.notification_arns, s3_uploader,
                            tags, parsed_args.fail_on_empty_changeset,
-                           parsed_args.disable_rollback)
+                           parsed_args.disable_rollback, getattr(parsed_globals, 'v2_debug', False))
 
     def deploy(self, deployer, stack_name, template_str,
                parameters, capabilities, execute_changeset, role_arn,
                notification_arns, s3_uploader, tags,
-               fail_on_empty_changeset=True, disable_rollback=False):
+               fail_on_empty_changeset=True, disable_rollback=False,
+               v2_debug=False):
         try:
             result = deployer.create_and_wait_for_changeset(
                 stack_name=stack_name,
@@ -339,10 +341,19 @@ def deploy(self, deployer, stack_name, template_str,
                 tags=tags
             )
         except exceptions.ChangeEmptyError as ex:
-            # TODO print the runtime check for cli v2 breakage. technically won't be breaking if --fail-on-empty-changeset is
-            # explicitly provided. but we cannot differentiate between whether fail-on-empty-changeset is true because it's default
-            # or because it's explicitly specified.
             if fail_on_empty_changeset:
+                if v2_debug:
+                    uni_print(
+                        'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, '
+                        'deploying an AWS CloudFormation Template that '
+                        'results in an empty changeset will NOT result in an '
+                        'error. You can add the -–no-fail-on-empty-changeset '
+                        'flag to migrate to v2 behavior and resolve this '
+                        'warning. See https://docs.aws.amazon.com/cli/latest/'
+                        'userguide/cliv2-migration-changes.html'
+                        '#cliv2-migration-cfn.\n',
+                        out_file=sys.stderr
+                    )
                 raise
             write_exception(ex, outfile=get_stdout_text_writer())
             return 0

awscli/customizations/globalargs.py

@@ -96,19 +96,129 @@ def resolve_cli_connect_timeout(parsed_args, session, **kwargs):
     _resolve_timeout(session, parsed_args, arg_name)
 
 def detect_migration_breakage(parsed_args, remaining_args, session, **kwargs):
-    if parsed_args.v2_debug:
-        url_params = [param for param in remaining_args if param.startswith('http://') or param.startswith('https://')]
-        if parsed_args.command == 'ecr' and remaining_args[0] == 'get-login':
-            uni_print('AWS CLI v2 MIGRATION WARNING: The ecr get-login command has been removed in AWS CLI v2. See https://docs.aws.amazon.com/cli/latest/userguide/cliv2-migration-changes.html#cliv2-migration-ecr-get-login.\n')
-        if url_params and session.full_config.get('cli_follow_urlparam', True):
-            uni_print('AWS CLI v2 MIGRATION WARNING: For input parameters that have a prefix of http:// or https://, AWS CLI v2 will no longer automatically request the content of the URL for the parameter, and the cli_follow_urlparam option has been removed. See https://docs.aws.amazon.com/cli/latest/userguide/cliv2-migration-changes.html#cliv2-migration-paramfile.\n')
-        for working, obsolete in HIDDEN_ALIASES.items():
-            working_split = working.split('.')
-            working_service = working_split[0]
-            working_cmd = working_split[1]
-            working_param = working_split[2]
-            if parsed_args.command == working_service and remaining_args[0] == working_cmd and f"--{working_param}" in remaining_args:
-                uni_print('AWS CLI v2 MIGRATION WARNING: You have entered command arguments that uses at least 1 of 21 hidden aliases that were removed in AWS CLI v2. See https://docs.aws.amazon.com/cli/latest/userguide/cliv2-migration-changes.html#cliv2-migration-aliases.\n')
+    if not parsed_args.v2_debug:
+        return
+    url_params = [
+        param for param in remaining_args
+        if param.startswith('http://') or param.startswith('https://')
+    ]
+    region = parsed_args.region or session.get_config_variable('region')
+    s3_config = session.get_config_variable('s3')
+    if 'PYTHONUTF8' in os.environ or 'PYTHONIOENCODING' in os.environ:
+        if 'AWS_CLI_FILE_ENCODING' not in os.environ:
+            uni_print(
+                'AWS CLI v2 MIGRATION WARNING: The PYTHONUTF8 and '
+                'PYTHONIOENCODING environment variables are unsupported '
+                'in AWS CLI v2. AWS CLI v2 uses AWS_CLI_FILE_ENCODING '
+                'instead, set this environment variable to resolve this. '
+                'See https://docs.aws.amazon.com/cli/latest/userguide/'
+                'cliv2-migration-changes.html'
+                '#cliv2-migration-encodingenvvar.\n',
+                out_file=sys.stderr
+            )
+    if (
+            s3_config is not None and s3_config
+                .get(
+                        'us_east_1_regional_endpoint',
+                        'legacy'
+                    ) == 'legacy' and region in ('us-east-1', None)
+    ):
+        session.register(
+            'request-created.s3.*',
+            warn_if_east_configured_global_endpoint
+        )
+    if session.get_config_variable('api_versions'):
+        uni_print(
+            'AWS CLI v2 MIGRATION WARNING: The AWS CLI v2 does not support '
+            'calling earlier versions of AWS service APIs via the '
+            '`api_versions` configuration file setting. To migrate to v2 '
+            'behavior and resolve this warning, remove the `api_versions` '
+            'setting in the configuration file. See '
+            'https://docs.aws.amazon.com/cli/latest/userguide/'
+            'cliv2-migration-changes.html#cliv2-migration-api-versions.\n',
+            out_file = sys.stderr
+        )
+    if session.full_config.get('plugins', {}):
+        uni_print(
+            'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, plugin support '
+            'is provisional. If you rely on plugins, be sure to lock into '
+            'a particular version of the AWS CLI and test the '
+            'functionality of your plugins for each upgrade. See '
+            'https://docs.aws.amazon.com/cli/latest/userguide/'
+            'cliv2-migration-changes.html#'
+            'cliv2-migration-profile-plugins\n',
+            out_file=sys.stderr
+        )
+    if parsed_args.command == 'ecr' and remaining_args[0] == 'get-login':
+        uni_print(
+            'AWS CLI v2 MIGRATION WARNING: The ecr get-login command has '
+            'been removed in AWS CLI v2. See https://docs.aws.amazon.com/'
+            'cli/latest/userguide/cliv2-migration-changes.html'
+            '#cliv2-migration-ecr-get-login.\n',
+            out_file=sys.stderr
+        )
+    if url_params and session.full_config.get('cli_follow_urlparam', True):
+        uni_print(
+            'AWS CLI v2 MIGRATION WARNING: For input parameters that have '
+            'a prefix of http:// or https://, AWS CLI v2 will no longer '
+            'automatically request the content of the URL for the '
+            'parameter, and the cli_follow_urlparam option has been '
+            'removed. See https://docs.aws.amazon.com/cli/latest/'
+            'userguide/cliv2-migration-changes.html'
+            '#cliv2-migration-paramfile.\n',
+            out_file=sys.stderr
+        )
+    for working, obsolete in HIDDEN_ALIASES.items():
+        working_split = working.split('.')
+        working_service = working_split[0]
+        working_cmd = working_split[1]
+        working_param = working_split[2]
+        if (
+                parsed_args.command == working_service
+                and remaining_args[0] == working_cmd
+                and f"--{working_param}" in remaining_args
+        ):
+            uni_print(
+                'AWS CLI v2 MIGRATION WARNING: You have entered command '
+                'arguments that uses at least 1 of 21 hidden aliases that '
+                'were removed in AWS CLI v2. See '
+                'https://docs.aws.amazon.com/cli/latest/userguide'
+                '/cliv2-migration-changes.html#cliv2-migration-aliases.\n',
+                out_file=sys.stderr
+            )
+    session.register('choose-signer.s3.*', warn_if_sigv2)
+
+def warn_if_east_configured_global_endpoint(request, operation_name, **kwargs):
+    # The regional us-east-1 endpoint is used in certain cases (e.g.
+    # FIPS/Dual-Stack is enabled). Rather than duplicating this logic
+    # from botocore, we check the endpoint URL directly.
+    if 's3.amazonaws.com' in request.url:
+        uni_print(
+            'AWS CLI v2 MIGRATION WARNING: When you configure AWS CLI v2 '
+            'to use the `us-east-1` region, it uses the true regional '
+            'endpoint rather than the global endpoint. To continue using '
+            'the global endpoint on v2, configure the region to be '
+            '`aws-global`. See https://docs.aws.amazon.com/cli/latest/'
+            'userguide/cliv2-migration-changes.html'
+            '#cliv2-migration-s3-regional-endpoint.\n',
+            out_file=sys.stderr
+        )
+
+def warn_if_sigv2(
+        signing_name,
+        region_name,
+        signature_version,
+        context,
+        **kwargs
+):
+    if context.get('auth_type', None) == 'v2':
+        uni_print(
+            'AWS CLI v2 MIGRATION WARNING: The AWS CLI v2 only uses Signature '
+            'v4 to authenticate Amazon S3 requests. Run the command `aws '
+            'configure set s3.signature_version s3v4` to migrate to v4 and '
+            'resolve this.\n',
+            out_file=sys.stderr
+        )
 
 def resolve_cli_read_timeout(parsed_args, session, **kwargs):
     arg_name = 'read_timeout'

awscli/customizations/paginate.py

@@ -135,6 +135,9 @@ def unify_paging_params(argument_table, operation_model, event_name,
     _remove_existing_paging_arguments(argument_table, paginator_config)
     parsed_args_event = event_name.replace('building-argument-table.',
                                            'operation-args-parsed.')
+    call_parameters_event = event_name.replace(
+        'building-argument-table', 'calling-command'
+    )
     shadowed_args = {}
     add_paging_argument(argument_table, 'starting-token',
                         PageArgument('starting-token', STARTING_TOKEN_HELP,
@@ -168,6 +171,14 @@ def unify_paging_params(argument_table, operation_model, event_name,
         partial(check_should_enable_pagination,
                 list(_get_all_cli_input_tokens(paginator_config)),
                 shadowed_args, argument_table))
+    session.register(
+        call_parameters_event,
+        partial(
+            check_should_enable_pagination_call_parameters,
+            session,
+            list(_get_all_input_tokens(paginator_config)),
+        ),
+    )
 
 
 def add_paging_argument(argument_table, arg_name, argument, shadowed_args):
@@ -240,6 +251,18 @@ def _get_all_cli_input_tokens(pagination_config):
         yield cli_name
 
 
+# Get all tokens but return them in API namespace rather than CLI namespace
+def _get_all_input_tokens(pagination_config):
+    # Get all input tokens including the limit_key
+    # if it exists.
+    tokens = _get_input_tokens(pagination_config)
+    for token_name in tokens:
+        yield token_name
+    if 'limit_key' in pagination_config:
+        key_name = pagination_config['limit_key']
+        yield key_name
+
+
 def _get_input_tokens(pagination_config):
     tokens = pagination_config['input_token']
     if not isinstance(tokens, list):
@@ -253,6 +276,45 @@ def _get_cli_name(param_objects, token_name):
             return param.cli_name.lstrip('-')
 
 
+def check_should_enable_pagination_call_parameters(
+        session,
+        input_tokens,
+        call_parameters,
+        parsed_args,
+        parsed_globals,
+        **kwargs
+):
+    """
+    Check for pagination args in the actual calling arguments passed to
+    the function.
+
+    If the user is using the --cli-input-json parameter to provide JSON
+    parameters they are all in the API naming space rather than the CLI
+    naming space and would be missed by the processing above. This function
+    gets called on the calling-command event.
+    """
+    if parsed_globals.v2_debug:
+        cli_input_json_data = session.emit_first_non_none_response(
+            f"get-cli-input-json-data",
+        )
+        if cli_input_json_data is None:
+            cli_input_json_data = {}
+        pagination_params_in_input_tokens = [
+            param for param in cli_input_json_data if param in input_tokens
+        ]
+        if pagination_params_in_input_tokens:
+            uni_print(
+                'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, if you specify '
+                'pagination parameters by using a file with the '
+                '`--cli-input-json` parameter, automatic pagination will be '
+                'turned off. This is not the case in v1. See '
+                'https://docs.aws.amazon.com/cli/latest/userguide/'
+                'cliv2-migration-changes.html'
+                '#cliv2-migration-skeleton-paging.\n',
+                out_file=sys.stderr
+            )
+
+
 class PageArgument(BaseCLIArgument):
     type_map = {
         'string': str,