Fixed #408 - tab value cause error in saneyaml.load()

 * Replace tab ('\t') with spaces to prevent saneyaml load error

Author: chinyeungli

src/attributecode/model.py

@@ -67,6 +67,7 @@
 from attributecode.util import filter_errors
 from attributecode.util import is_valid_name
 from attributecode.util import on_windows
+from attributecode.util import replace_tab_with_spaces
 from attributecode.util import wrap_boolean_value
 from attributecode.util import UNC_PREFIX
 from attributecode.util import ungroup_licenses
@@ -934,7 +935,8 @@ def load(self, location):
             # The 'Yes' and 'No' will be converted to 'True' and 'False' in the yaml.load()
             # Therefore, we need to wrap the original value in quote to prevent
             # the conversion
-            input = wrap_boolean_value(input_text)
+            pre_input = wrap_boolean_value(input_text)
+            input = replace_tab_with_spaces(pre_input)
             # FIXME: this should be done in the commands, not here
             """
             The running_inventory defines if the current process is 'inventory' or not.

src/attributecode/util.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf8 -*-
 # ============================================================================
-#  Copyright (c) 2013-2018 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Copyright (c) 2013-2019 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -140,6 +140,15 @@ def wrap_boolean_value(context):
             updated_context += line + '\n'
     return updated_context
 
+def replace_tab_with_spaces(context):
+    updated_context = ''
+    for line in context.splitlines():
+        """
+        Replace tab with 4 spaces
+        """
+        updated_context += line.replace('\t', '    ') + '\n'
+    return updated_context
+
 # TODO: rename to normalize_path
 def get_absolute(location):
     """

tests/test_model.py

@@ -2,7 +2,7 @@
 # -*- coding: utf8 -*-
 
 # ============================================================================
-#  Copyright (c) 2014-2018 nexB Inc. http://www.nexb.com/ - All rights reserved.
+#  Copyright (c) 2014-2019 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
@@ -37,6 +37,7 @@
 from attributecode.util import add_unc
 from attributecode.util import load_csv
 from attributecode.util import to_posix
+from attributecode.util import replace_tab_with_spaces
 
 from testing_utils import extract_test_loc
 from testing_utils import get_temp_file
@@ -303,6 +304,19 @@ def test_saneyaml_load_can_parse_verbatim_text_unstripped(self):
 
         assert expected == list(result.items())
 
+    def test_saneyaml_load_can_parse_verbatim_tab_text_unstripped(self):
+        test = get_test_content('test_model/parse/continuation_verbatim_with_tab.about')
+        data = replace_tab_with_spaces(test)
+        result = saneyaml.load(data)
+
+        expected = [
+            (u'single_line', u'optional'),
+            (u'other_field', u'value'),
+            (u'multi_line', u'This is a long description\nwith tab.\n')
+        ]
+
+        assert expected == list(result.items())
+
     def test_saneyaml_load_report_error_for_invalid_field_name(self):
         test = get_test_content('test_model/parse/invalid_names.about')
         try:
@@ -699,8 +713,6 @@ def test_About_dumps_does_all_non_empty_present_fields(self):
             Error(INFO, 'Field custom2 is a custom field.'),
             Error(INFO, 'Field custom2 is present but empty.')
         ]
-        print("########################################")
-        print(a.errors)
         assert sorted(expected_error) == sorted(a.errors)
 
         expected = '''about_resource: .

tests/testdata/test_model/parse/continuation_verbatim_with_tab.about

@@ -0,0 +1,7 @@
+single_line: optional
+other_field: value
+ 
+
+multi_line: |
+	This is a long description
+	with tab.