Merge pull request #7787 from hssyoo/change-cfn-yml-spec

Use YAML 1.1 spec for CFN

Author: hssyoo

.changes/next-release/bugfix-cloudformation-20253.json

@@ -0,0 +1,5 @@
+{
+  "type": "bugfix",
+  "category": "cloudformation",
+  "description": "Fixes `#3991 <https://github.com/aws/aws-cli/issues/3991>`__. Use YAML 1.1 spec in alignment with CloudFormation YAML support."
+}

awscli/customizations/cloudformation/yamlhelper.py

@@ -63,21 +63,17 @@ def _dict_representer(dumper, data):
     return dumper.represent_dict(data.items())
 
 
-def _add_yaml_1_1_boolean_resolvers(resolver_cls):
-    # CloudFormation treats unquoted values that are YAML 1.1 native
-    # booleans as booleans, rather than strings. In YAML 1.2, the only
-    # boolean values are "true" and "false" so values such as "yes" and "no"
-    # when loaded as strings are not quoted when dumped. This logic ensures
-    # that we dump these values with quotes so that CloudFormation treats
-    # these values as strings and not booleans.
-    boolean_regex = re.compile(
-        '^(?:yes|Yes|YES|no|No|NO'
-        '|true|True|TRUE|false|False|FALSE'
-        '|on|On|ON|off|Off|OFF)$'
-    )
-    boolean_first_chars = list(u'yYnNtTfFoO')
-    resolver_cls.add_implicit_resolver(
-        'tag:yaml.org,2002:bool', boolean_regex, boolean_first_chars)
+def _str_representer(dumper, data):
+    # ruamel removes quotes from values that are unambiguously strings.
+    # Values like 0888888 can only be a string because integers can't
+    # have leading 0s and octals can't have the digits 8 and 9.
+    # However, CloudFormation treats these nonoctal values as integers
+    # and removes the leading 0s. This logic ensures that nonoctal
+    # values are quoted when dumped.
+    style = None
+    if re.match('^0[0-9]*[89][0-9]*$', data):
+        style = "'"
+    return dumper.represent_scalar('tag:yaml.org,2002:str', data, style=style)
 
 
 def yaml_dump(dict_to_dump):
@@ -88,10 +84,11 @@ def yaml_dump(dict_to_dump):
     """
 
     yaml = ruamel.yaml.YAML(typ="safe", pure=True)
+    yaml.version = (1, 1)
     yaml.default_flow_style = False
     yaml.Representer = FlattenAliasRepresenter
-    _add_yaml_1_1_boolean_resolvers(yaml.Resolver)
     yaml.Representer.add_representer(OrderedDict, _dict_representer)
+    yaml.Representer.add_representer(str, _str_representer)
 
     return dump_yaml_to_str(yaml, dict_to_dump)
 
@@ -111,12 +108,12 @@ def yaml_parse(yamlstr):
         return json.loads(yamlstr, object_pairs_hook=OrderedDict)
     except ValueError:
         yaml = ruamel.yaml.YAML(typ="safe", pure=True)
+        yaml.version = (1, 1)
         yaml.Constructor.add_constructor(
             ruamel.yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
             _dict_constructor)
         yaml.Constructor.add_multi_constructor(
             "!", intrinsics_multi_constructor)
-        _add_yaml_1_1_boolean_resolvers(yaml.Resolver)
 
         return yaml.load(yamlstr)
 

awscli/testutils.py

@@ -34,6 +34,7 @@
 from pprint import pformat
 from subprocess import Popen, PIPE
 import unittest
+import re
 
 from awscli.compat import StringIO
 
@@ -58,6 +59,7 @@
 import awscli.clidriver
 from awscli.plugin import load_plugins
 from awscli.clidriver import CLIDriver
+from awscli.customizations.cloudformation.yamlhelper import yaml_dump
 
 
 _LOADER = botocore.loaders.Loader()
@@ -1012,3 +1014,8 @@ def wait(self, check, *args, **kwargs):
     def _fail_message(self, attempts, successes):
         format_args = (attempts, successes)
         return 'Failed after %s attempts, only had %s successes' % format_args
+
+
+def yaml_dump_without_header(dict_to_dump):
+    dumped_yaml = yaml_dump(dict_to_dump)
+    return re.sub('%YAML 1.1\n---(\n| )', '', dumped_yaml)

tests/functional/cloudformation/test_package.py

@@ -23,7 +23,7 @@
 from awscli.customizations.cloudformation.artifact_exporter import make_zip
 from awscli.customizations.cloudformation.yamlhelper import yaml_dump
 from awscli.customizations.cloudformation.artifact_exporter import Template
-from awscli.testutils import skip_if_windows
+from awscli.testutils import yaml_dump_without_header, skip_if_windows
 
 
 class TestPackageZipFiles(TestCase):
@@ -87,7 +87,7 @@ def _generate_template_cases():
 def test_known_templates(input_template, output_template):
     template = Template(input_template, os.getcwd(), None)
     exported = template.export()
-    result = yaml_dump(exported)
+    result = yaml_dump_without_header(exported)
     expected = open(output_template, 'r').read()
 
     assert result == expected, (

tests/unit/customizations/cloudformation/test_yamlhelper.py

@@ -20,6 +20,7 @@
 from awscli.customizations.cloudformation.deployer import Deployer
 from awscli.customizations.cloudformation.yamlhelper import yaml_parse, yaml_dump
 from tests.unit.customizations.cloudformation import BaseYAMLTest
+from awscli.testutils import yaml_dump_without_header
 
 
 class TestYaml(BaseYAMLTest):
@@ -145,7 +146,7 @@ def test_parse_yaml_preserve_elements_order(self):
         ])
         self.assertEqual(expected_dict, output_dict)
 
-        output_template = yaml_dump(output_dict)
+        output_template = yaml_dump_without_header(output_dict)
         self.assertEqual(input_template, output_template)
 
     def test_yaml_merge_tag(self):
@@ -182,7 +183,7 @@ def test_unroll_yaml_anchors(self):
             '      Foo: bar\n'
             '      Spam: eggs\n'
         )
-        actual = yaml_dump(template)
+        actual = yaml_dump_without_header(template)
         self.assertEqual(actual, expected)
 
     def test_yaml_dump_quotes_boolean_strings(self):
@@ -193,4 +194,26 @@ def test_yaml_dump_quotes_boolean_strings(self):
         ]
         for bool_as_string in bools_as_strings:
             self.assertEqual(
-                yaml_dump(bool_as_string), "'%s'\n" % bool_as_string)
+                yaml_dump_without_header(bool_as_string),
+                "'%s'\n" % bool_as_string)
+
+    def test_yaml_dump_quotes_octal_strings(self):
+        octals_as_strings = [
+            '01111111', '02222222', '03333333',
+            '04444444', '05555555', '06666666',
+            '07777777', '08888888', '09999999'
+        ]
+        for octal_as_string in octals_as_strings:
+            self.assertEqual(
+                yaml_dump_without_header(octal_as_string),
+                "'%s'\n" % octal_as_string)
+
+    def test_yaml_dump_include_1_1_header(self):
+        template = {"Resources": {"Foo": "Bar"}}
+        expected = (
+            '%YAML 1.1\n---\n'
+            'Resources:\n'
+            '  Foo: Bar\n'
+        )
+        actual = yaml_dump(template)
+        self.assertEqual(actual, expected)