Use new saneyaml library

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

setup.py

@@ -68,9 +68,15 @@ def read(*names, **kwargs):
     ],
     install_requires=[
         'jinja2 >= 2.9, < 3.0',
+
         'click >= 6.7, < 7.0',
+
         "backports.csv ; python_version<'3.6'",
-        'PyYAML >= 3.0, < 4.0',
+
+        # required by saneyaml
+        'PyYAML >= 3.11, <=3.13',
+        'saneyaml',
+
         'boolean.py >= 3.5, < 4.0',
         'license_expression >= 0.94, < 1.0',
     ],

src/attributecode/__init__.py

@@ -24,11 +24,12 @@
 
 try:
     # Python 2
-    unicode
+    unicode  # NOQA
 except NameError:  # pragma: nocover
-    # Python 3 
-    unicode = str  #NOQA
+    # Python 3
+    unicode = str  # NOQA
 
+import saneyaml
 
 __version__ = '3.3.0'
 

src/attributecode/model.py

@@ -36,7 +36,6 @@
 import posixpath
 import re
 
-import yaml
 
 from attributecode.util import python2
 
@@ -180,7 +179,7 @@ def serialize(self):
                     # insert 4 spaces for newline values
                     value = u'    '.join(value)
             else:
-                # See https://github.com/nexB/aboutcode-toolkit/issues/323
+                # FIXME: See https://github.com/nexB/aboutcode-toolkit/issues/323
                 # The yaml.load() will throw error if the parsed value
                 # contains ': ' character. A work around is to put a pipe, '|'
                 # to indicate the whole value as a string
@@ -1017,7 +1016,7 @@ def load(self, location, mapping_file=None):
             with codecs.open(loc, encoding='utf-8') as txt:
                 input_text = txt.read()
             # Check for duplicated key
-            yaml.load(input_text, Loader=util.NoDuplicateLoader)
+            saneyaml.load(input_text, allow_duplicate_keys=False)
             """
             The running_inventory defines if the current process is 'inventory' or not.
             This is used for the validation of the path of the 'about_resource'.
@@ -1028,10 +1027,12 @@ def load(self, location, mapping_file=None):
             and then join with the 'about_resource'
             """
             running_inventory = True
+            # FIXME: why??
             # wrap the value of the boolean field in quote to avoid
             # automatically conversion from yaml.load
             input = util.wrap_boolean_value(input_text)  # NOQA
-            errs = self.load_dict(saneyaml.load(input), base_dir, running_inventory, mapping_file)
+            data = saneyaml.load(input, allow_duplicate_keys=False)
+            errs = self.load_dict(data, base_dir, running_inventory, mapping_file)
             errors.extend(errs)
         except Exception as e:
             msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n' + str(e)

src/attributecode/saneyaml.py

@@ -245,7 +245,7 @@ def norm(p):
 def to_native(path):
     """
     Return a path using the current OS path separator given a path that may
-    contain posix or windows separators, converting "/" to "\\" on windows 
+    contain posix or windows separators, converting "/" to "\\" on windows
     and "\\" to "/" on posix OSes.
     """
     path = path.replace(ntpath.sep, os.path.sep)
@@ -532,7 +532,7 @@ def have_network_connection():
     else:
         import http.client as httplib  # NOQA
 
-    http_connection = httplib.HTTPConnection('dejacode.org', timeout=10)
+    http_connection = httplib.HTTPConnection('dejacode.org', timeout=10)  # NOQA
     try:
         http_connection.connect()
     except socket.error:
@@ -605,7 +605,7 @@ def copy_license_notice_files(fields, base_dir, reference_dir, afp):
     where reference license an notice files are stored and the `afp`
     about_file_path value, this function will copy to the base_dir the
     license_file or notice_file if found in the reference_dir
-    
+
     """
     lic_name = ''
     for key, value in fields:
@@ -769,66 +769,3 @@ def unique(sequence):
         if item not in deduped:
             deduped.append(item)
     return deduped
-
-
-# FIXME: remove and replace by saneyaml
-from collections import Hashable
-
-from yaml.reader import Reader
-from yaml.scanner import Scanner
-from yaml.parser import Parser
-from yaml.composer import Composer
-from yaml.constructor import Constructor, ConstructorError
-from yaml.resolver import Resolver
-from yaml.nodes import MappingNode
-
-# FIXME: add docstring
-class NoDuplicateConstructor(Constructor):
-    def construct_mapping(self, node, deep=False):
-        if not isinstance(node, MappingNode):
-            raise ConstructorError(
-                None, None,
-                "expected a mapping node, but found %s" % node.id,
-                node.start_mark)
-        mapping = {}
-        for key_node, value_node in node.value:
-            # keys can be list -> deep
-            key = self.construct_object(key_node, deep=True)
-            # lists are not hashable, but tuples are
-            if not isinstance(key, Hashable):
-                if isinstance(key, list):
-                    key = tuple(key)
-
-            if sys.version_info.major == 2:
-                try:
-                    hash(key)
-                except TypeError as exc:
-                    raise ConstructorError(
-                        "while constructing a mapping", node.start_mark,
-                        "found unacceptable key (%s)" %
-                        exc, key_node.start_mark)
-            else:
-                if not isinstance(key, Hashable):
-                    raise ConstructorError(
-                        "while constructing a mapping", node.start_mark,
-                        "found unhashable key", key_node.start_mark)
-
-            value = self.construct_object(value_node, deep=deep)
-
-            # Actually do the check.
-            if key in mapping:
-                raise KeyError("Got duplicate key: {!r}".format(key))
-
-            mapping[key] = value
-        return mapping
-
-
-# FIXME: add docstring
-class NoDuplicateLoader(Reader, Scanner, Parser, Composer, NoDuplicateConstructor, Resolver):
-    def __init__(self, stream):
-        Reader.__init__(self, stream)
-        Scanner.__init__(self)
-        Parser.__init__(self)
-        Composer.__init__(self)
-        NoDuplicateConstructor.__init__(self)
-        Resolver.__init__(self)