Implement v2-debug checks for args, config, customizations, s3, and environment variables

awscli/clidriver.py

@@ -41,6 +41,7 @@
 )
 from awscli.commands import CLICommand
 from awscli.compat import get_stderr_text_writer
+from awscli.customizations.utils import uni_print
 from awscli.formatter import get_formatter
 from awscli.help import (
     OperationHelpCommand,
@@ -79,8 +80,6 @@ def main():
 def create_clidriver():
     session = botocore.session.Session(EnvironmentVariables)
     _set_user_agent_for_session(session)
-    # TODO check if full config plugins is empty or not. if it's not, we signal the warning for plugin support being provisional
-    # similarly, we check for api_versions config value here.
     load_plugins(
         session.full_config.get('plugins', {}),
         event_hooks=session.get_component('event_emitter'),
@@ -547,6 +546,12 @@ def __call__(self, args, parsed_globals):
             parsed_args, self.arg_table
         )
 
+        self._detect_migration_breakage(
+            self._session,
+            parsed_args,
+            parsed_globals,
+            self.arg_table
+        )
         event = f'calling-command.{self._parent_name}.{self._name}'
         override = self._emit_first_non_none_response(
             event,
@@ -663,6 +668,39 @@ def _create_operation_parser(self, arg_table):
         parser = ArgTableArgParser(arg_table)
         return parser
 
+    def _detect_migration_breakage(self, session, parsed_args, parsed_globals, arg_table):
+        if (
+                session.get_scoped_config()
+                        .get('cli_binary_format', None) == 'raw-in-base64-out'
+        ):
+            # if cli_binary_format is set to raw-in-base64-out, then v2 behavior will
+            # be the same as v1, so there is no breaking change in this case.
+            return
+        if parsed_globals.v2_debug:
+            parsed_args_to_check = {
+                arg: getattr(parsed_args, arg)
+                for arg in vars(parsed_args) if getattr(parsed_args, arg)
+            }
+
+            arg_values_to_check = [
+                arg.py_name for arg in arg_table.values()
+                if arg.py_name in parsed_args_to_check
+                   and arg.argument_model.type_name == 'blob'
+                   and parsed_args_to_check[arg.py_name].startswith('file://')
+            ]
+            if arg_values_to_check:
+                uni_print(
+                    'AWS CLI v2 MIGRATION WARNING: When specifying a blob-type '
+                    'parameter starting with `file://`, AWS CLI v2 will assume '
+                    'the content of the file is already base64-encoded. To '
+                    'maintain v1 behavior after upgrading to v2, set the '
+                    '`cli_binary_format` configuration variable to '
+                    '`raw-in-base64-out`. See https://docs.aws.amazon.com/cli/'
+                    'latest/userguide/cliv2-migration-changes.html#'
+                    'cliv2-migration-binaryparam.\n',
+                    out_file=sys.stderr
+                )
+
 
 class CLIOperationCaller:
     """Call an AWS operation and format the response."""

awscli/customizations/cloudformation/deploy.py

@@ -24,6 +24,7 @@
 
 from awscli.customizations.commands import BasicCommand
 from awscli.compat import get_stdout_text_writer
+from awscli.customizations.utils import uni_print
 from awscli.utils import create_nested_client, write_exception
 
 LOG = logging.getLogger(__name__)
@@ -321,12 +322,13 @@ def _run_main(self, parsed_args, parsed_globals):
                            parsed_args.execute_changeset, parsed_args.role_arn,
                            parsed_args.notification_arns, s3_uploader,
                            tags, parsed_args.fail_on_empty_changeset,
-                           parsed_args.disable_rollback)
+                           parsed_args.disable_rollback, getattr(parsed_globals, 'v2_debug', False))
 
     def deploy(self, deployer, stack_name, template_str,
                parameters, capabilities, execute_changeset, role_arn,
                notification_arns, s3_uploader, tags,
-               fail_on_empty_changeset=True, disable_rollback=False):
+               fail_on_empty_changeset=True, disable_rollback=False,
+               v2_debug=False):
         try:
             result = deployer.create_and_wait_for_changeset(
                 stack_name=stack_name,
@@ -339,10 +341,19 @@ def deploy(self, deployer, stack_name, template_str,
                 tags=tags
             )
         except exceptions.ChangeEmptyError as ex:
-            # TODO print the runtime check for cli v2 breakage. technically won't be breaking if --fail-on-empty-changeset is
-            # explicitly provided. but we cannot differentiate between whether fail-on-empty-changeset is true because it's default
-            # or because it's explicitly specified.
             if fail_on_empty_changeset:
+                if v2_debug:
+                    uni_print(
+                        'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, '
+                        'deploying an AWS CloudFormation Template that '
+                        'results in an empty changeset will NOT result in an '
+                        'error. You can add the -–no-fail-on-empty-changeset '
+                        'flag to migrate to v2 behavior and resolve this '
+                        'warning. See https://docs.aws.amazon.com/cli/latest/'
+                        'userguide/cliv2-migration-changes.html'
+                        '#cliv2-migration-cfn.\n',
+                        out_file=sys.stderr
+                    )
                 raise
             write_exception(ex, outfile=get_stdout_text_writer())
             return 0

awscli/customizations/globalargs.py

@@ -97,18 +97,127 @@ def resolve_cli_connect_timeout(parsed_args, session, **kwargs):
 
 def detect_migration_breakage(parsed_args, remaining_args, session, **kwargs):
     if parsed_args.v2_debug:
-        url_params = [param for param in remaining_args if param.startswith('http://') or param.startswith('https://')]
+        url_params = [
+            param for param in remaining_args
+            if param.startswith('http://') or param.startswith('https://')
+        ]
+        region = parsed_args.region or session.get_config_variable('region')
+        s3_config = session.get_config_variable('s3')
+        if 'PYTHONUTF8' in os.environ or 'PYTHONIOENCODING' in os.environ:
+            if 'AWS_CLI_FILE_ENCODING' not in os.environ:
+                uni_print(
+                    'AWS CLI v2 MIGRATION WARNING: The PYTHONUTF8 and '
+                    'PYTHONIOENCODING environment variables are unsupported '
+                    'in AWS CLI v2. AWS CLI v2 uses AWS_CLI_FILE_ENCODING '
+                    'instead, set this environment variable to resolve this. '
+                    'See https://docs.aws.amazon.com/cli/latest/userguide/'
+                    'cliv2-migration-changes.html'
+                    '#cliv2-migration-encodingenvvar.\n',
+                    out_file=sys.stderr
+                )
+        if (
+                s3_config is not None and s3_config
+                    .get(
+                            'us_east_1_regional_endpoint',
+                            'legacy'
+                        ) == 'legacy' and region in ('us-east-1', None)
+        ):
+            session.register(
+                'request-created.s3.*',
+                warn_if_east_configured_global_endpoint
+            )
+        if session.get_config_variable('api_versions'):
+            uni_print(
+                'AWS CLI v2 MIGRATION WARNING: The AWS CLI v2 does not support '
+                'calling earlier versions of AWS service APIs via the '
+                '`api_versions` configuration file setting. To migrate to v2 '
+                'behavior and resolve this warning, remove the `api_versions` '
+                'setting in the configuration file. See '
+                'https://docs.aws.amazon.com/cli/latest/userguide/'
+                'cliv2-migration-changes.html#cliv2-migration-api-versions.\n',
+                out_file = sys.stderr
+            )
+        if session.full_config.get('plugins', {}):
+            uni_print(
+                'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, plugin support '
+                'is provisional. If you rely on plugins, be sure to lock into '
+                'a particular version of the AWS CLI and test the '
+                'functionality of your plugins for each upgrade. See '
+                'https://docs.aws.amazon.com/cli/latest/userguide/'
+                'cliv2-migration-changes.html#'
+                'cliv2-migration-profile-plugins\n',
+                out_file=sys.stderr
+            )
         if parsed_args.command == 'ecr' and remaining_args[0] == 'get-login':
-            uni_print('AWS CLI v2 MIGRATION WARNING: The ecr get-login command has been removed in AWS CLI v2. See https://docs.aws.amazon.com/cli/latest/userguide/cliv2-migration-changes.html#cliv2-migration-ecr-get-login.\n')
+            uni_print(
+                'AWS CLI v2 MIGRATION WARNING: The ecr get-login command has '
+                'been removed in AWS CLI v2. See https://docs.aws.amazon.com/'
+                'cli/latest/userguide/cliv2-migration-changes.html'
+                '#cliv2-migration-ecr-get-login.\n',
+                out_file=sys.stderr
+            )
         if url_params and session.full_config.get('cli_follow_urlparam', True):
-            uni_print('AWS CLI v2 MIGRATION WARNING: For input parameters that have a prefix of http:// or https://, AWS CLI v2 will no longer automatically request the content of the URL for the parameter, and the cli_follow_urlparam option has been removed. See https://docs.aws.amazon.com/cli/latest/userguide/cliv2-migration-changes.html#cliv2-migration-paramfile.\n')
+            uni_print(
+                'AWS CLI v2 MIGRATION WARNING: For input parameters that have '
+                'a prefix of http:// or https://, AWS CLI v2 will no longer '
+                'automatically request the content of the URL for the '
+                'parameter, and the cli_follow_urlparam option has been '
+                'removed. See https://docs.aws.amazon.com/cli/latest/'
+                'userguide/cliv2-migration-changes.html'
+                '#cliv2-migration-paramfile.\n',
+                out_file=sys.stderr
+            )
         for working, obsolete in HIDDEN_ALIASES.items():
             working_split = working.split('.')
             working_service = working_split[0]
             working_cmd = working_split[1]
             working_param = working_split[2]
-            if parsed_args.command == working_service and remaining_args[0] == working_cmd and f"--{working_param}" in remaining_args:
-                uni_print('AWS CLI v2 MIGRATION WARNING: You have entered command arguments that uses at least 1 of 21 hidden aliases that were removed in AWS CLI v2. See https://docs.aws.amazon.com/cli/latest/userguide/cliv2-migration-changes.html#cliv2-migration-aliases.\n')
+            if (
+                    parsed_args.command == working_service
+                    and remaining_args[0] == working_cmd
+                    and f"--{working_param}" in remaining_args
+            ):
+                uni_print(
+                    'AWS CLI v2 MIGRATION WARNING: You have entered command '
+                    'arguments that uses at least 1 of 21 hidden aliases that '
+                    'were removed in AWS CLI v2. See '
+                    'https://docs.aws.amazon.com/cli/latest/userguide'
+                    '/cliv2-migration-changes.html#cliv2-migration-aliases.\n',
+                    out_file=sys.stderr
+                )
+        session.register('choose-signer.s3.*', warn_if_sigv2)
+
+def warn_if_east_configured_global_endpoint(request, operation_name, **kwargs):
+    # The regional us-east-1 endpoint is used in certain cases (e.g.
+    # FIPS/Dual-Stack is enabled). Rather than duplicating this logic
+    # from botocore, we check the endpoint URL directly.
+    if 's3.amazonaws.com' in request.url:
+        uni_print(
+            'AWS CLI v2 MIGRATION WARNING: When you configure AWS CLI v2 '
+            'to use the `us-east-1` region, it uses the true regional '
+            'endpoint rather than the global endpoint. To continue using '
+            'the global endpoint on v2, configure the region to be '
+            '`aws-global`. See https://docs.aws.amazon.com/cli/latest/'
+            'userguide/cliv2-migration-changes.html'
+            '#cliv2-migration-s3-regional-endpoint.\n',
+            out_file=sys.stderr
+        )
+
+def warn_if_sigv2(
+        signing_name,
+        region_name,
+        signature_version,
+        context,
+        **kwargs
+):
+    if context.get('auth_type', None) == 'v2':
+        uni_print(
+            'AWS CLI v2 MIGRATION WARNING: The AWS CLI v2 only uses Signature '
+            'v4 to authenticate Amazon S3 requests. Run the command `aws '
+            'configure set s3.signature_version s3v4` to migrate to v4 and '
+            'resolve this.\n',
+            out_file=sys.stderr
+        )
 
 def resolve_cli_read_timeout(parsed_args, session, **kwargs):
     arg_name = 'read_timeout'

awscli/customizations/paginate.py

@@ -135,6 +135,9 @@ def unify_paging_params(argument_table, operation_model, event_name,
     _remove_existing_paging_arguments(argument_table, paginator_config)
     parsed_args_event = event_name.replace('building-argument-table.',
                                            'operation-args-parsed.')
+    call_parameters_event = event_name.replace(
+        'building-argument-table', 'calling-command'
+    )
     shadowed_args = {}
     add_paging_argument(argument_table, 'starting-token',
                         PageArgument('starting-token', STARTING_TOKEN_HELP,
@@ -168,6 +171,13 @@ def unify_paging_params(argument_table, operation_model, event_name,
         partial(check_should_enable_pagination,
                 list(_get_all_cli_input_tokens(paginator_config)),
                 shadowed_args, argument_table))
+    session.register(
+        call_parameters_event,
+        partial(
+            check_should_enable_pagination_call_parameters,
+            list(_get_all_input_tokens(paginator_config)),
+        ),
+    )
 
 
 def add_paging_argument(argument_table, arg_name, argument, shadowed_args):
@@ -240,6 +250,18 @@ def _get_all_cli_input_tokens(pagination_config):
         yield cli_name
 
 
+# Get all tokens but return them in API namespace rather than CLI namespace
+def _get_all_input_tokens(pagination_config):
+    # Get all input tokens including the limit_key
+    # if it exists.
+    tokens = _get_input_tokens(pagination_config)
+    for token_name in tokens:
+        yield token_name
+    if 'limit_key' in pagination_config:
+        key_name = pagination_config['limit_key']
+        yield key_name
+
+
 def _get_input_tokens(pagination_config):
     tokens = pagination_config['input_token']
     if not isinstance(tokens, list):
@@ -253,6 +275,32 @@ def _get_cli_name(param_objects, token_name):
             return param.cli_name.lstrip('-')
 
 
+# This function checks for pagination args in the actual calling
+# arguments passed to the function.  If the user is using the
+# --cli-input-json parameter to provide JSON parameters they are
+# all in the API naming space rather than the CLI naming space
+# and would be missed by the processing above.  This function gets
+# called on the calling-command event.
+def check_should_enable_pagination_call_parameters(
+    input_tokens, call_parameters, parsed_args, parsed_globals, **kwargs
+):
+    if parsed_globals.v2_debug:
+        pagination_params_in_input_tokens = [
+            param for param in call_parameters if param in input_tokens
+        ]
+        if pagination_params_in_input_tokens:
+            uni_print(
+                'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, if you specify '
+                'pagination parameters by using a file with the '
+                '`--cli-input-json` parameter, automatic pagination will be '
+                'turned off. This is not the case in v1. See '
+                'https://docs.aws.amazon.com/cli/latest/userguide/'
+                'cliv2-migration-changes.html'
+                '#cliv2-migration-skeleton-paging.\n',
+                out_file=sys.stderr
+            )
+
+
 class PageArgument(BaseCLIArgument):
     type_map = {
         'string': str,

awscli/customizations/s3/subcommands.py

@@ -775,7 +775,7 @@ def _run_main(self, parsed_args, parsed_globals):
                                   runtime_config)
         cmd.set_clients()
         cmd.create_instructions()
-        return cmd.run()
+        return cmd.run(getattr(parsed_globals, 'v2_debug', False))
 
     def _build_call_parameters(self, args, command_params):
         """
@@ -1019,7 +1019,7 @@ def choose_sync_strategies(self):
 
         return sync_strategies
 
-    def run(self):
+    def run(self, v2_debug=False):
         """
         This function wires together all of the generators and completes
         the command.  First a dictionary is created that is indexed first by
@@ -1056,6 +1056,21 @@ def run(self):
         result_queue = queue.Queue()
         operation_name = cmd_translation[paths_type]
 
+        if v2_debug:
+            if operation_name == 'copy':
+                uni_print(
+                    'AWS CLI v2 MIGRATION WARNING: In AWS CLI v2, object '
+                    'properties will be copied from the source in multipart '
+                    'copies between S3 buckets. This may result in extra S3 '
+                    'API calls being made. Breakage may occur if the principal '
+                    'does not have permission to call these extra APIs. This '
+                    'warning cannot be resolved. See '
+                    'https://docs.aws.amazon.com/cli/latest/userguide/'
+                    'cliv2-migration-changes.html'
+                    '#cliv2-migration-s3-copy-metadata\n\n',
+                    out_file=sys.stderr
+                )
+
         fgen_kwargs = {
             'client': self._source_client, 'operation_name': operation_name,
             'follow_symlinks': self.parameters['follow_symlinks'],