Fixed #531 - transform to work with nested list

 * The `transform` will now work with nested list (specially for json formatted input)

Signed-off-by: Chin Yeung Li <[email protected]>

Author: chinyeungli

src/attributecode/transform.py

@@ -26,6 +26,7 @@
 from attributecode.util import csv
 from attributecode.util import replace_tab_with_spaces
 
+
 def transform_csv(location):
     """
     Read a CSV file at `location` and convert data into list of dictionaries.
@@ -109,7 +110,7 @@ def normalize_dict_data(data):
     """
     try:
         # Check if this is a JSON output from scancode-toolkit
-        if(data["headers"][0]["tool_name"] == "scancode-toolkit"):
+        if (data["headers"][0]["tool_name"] == "scancode-toolkit"):
             # only takes data inside "files"
             new_data = data["files"]
     except:
@@ -129,10 +130,12 @@ def transform_data(data, transformer):
     renamed_field_data = transformer.apply_renamings(data)
 
     if transformer.field_filters:
-        renamed_field_data = list(transformer.filter_fields(renamed_field_data))
+        renamed_field_data = list(
+            transformer.filter_fields(renamed_field_data))
 
     if transformer.exclude_fields:
-        renamed_field_data = list(transformer.filter_excluded(renamed_field_data))
+        renamed_field_data = list(
+            transformer.filter_excluded(renamed_field_data))
 
     errors = transformer.check_required_fields(renamed_field_data)
     if errors:
@@ -277,23 +280,26 @@ def apply_renamings(self, data):
         based on this Transformer configuration.
         """
         renamings = self.field_renamings
+        renamed_to_list = list(renamings.keys())
+        renamed_from_list = list(renamings.values())
         if not renamings:
             return data
-        renamings = {n: rn for n, rn in renamings.items()}
-
-        renamed_list = []
-        for row in data:
-            renamed = {}
-            for key in row:
-                matched = False
-                for renamed_key in renamings:
-                    if key == renamings[renamed_key]:
-                        renamed[renamed_key] = row[key]
-                        matched = True
-                if not matched:
-                    renamed[key] = row[key]
-            renamed_list.append(renamed)
-        return renamed_list
+        if isinstance(data, dict):
+            renamed_obj = {}
+            for key, value in data.items():
+                if key in renamed_from_list:
+                    for idx, renamed_from_key in enumerate(renamed_from_list):
+                        if key == renamed_from_key:
+                            renamed_key = renamed_to_list[idx]
+                            renamed_obj[renamed_key] = self.apply_renamings(
+                                value)
+                else:
+                    renamed_obj[key] = self.apply_renamings(value)
+            return renamed_obj
+        elif isinstance(data, list):
+            return [self.apply_renamings(item) for item in data]
+        else:
+            return data
 
     """
     def clean_fields(self, field_names):
@@ -324,8 +330,18 @@ def filter_excluded(self, data):
         """
         # exclude_fields = set(self.clean_fields(self.exclude_fields))
         exclude_fields = set(self.exclude_fields)
+        filtered_list = []
         for entry in data:
-            yield {k: v for k, v in entry.items() if k not in exclude_fields}
+            result = {}
+            for k, v in entry.items():
+                if type(v) == list:
+                    result[k] = self.filter_excluded(v)
+                elif k not in exclude_fields:
+                    result[k] = v
+            filtered_list.append(result)
+            # yield result
+            # yield {k: v for k, v in entry.items() if k not in exclude_fields}
+        return filtered_list
 
 
 def check_duplicate_fields(field_names):
@@ -373,6 +389,7 @@ def write_json(location, data):
     with open(location, 'w') as jsonfile:
         json.dump(data, jsonfile, indent=3)
 
+
 def read_excel(location, worksheet=None):
     """
     Read XLSX at `location`, return a list of ordered dictionaries, one

tests/test_transform.py

@@ -57,7 +57,8 @@ def test_transform_data(self):
         data, err = transform_data(data, transformer)
 
         expect_name = [u'about_resource', u'name', u'version']
-        expected_data = [dict(OrderedDict([(u'about_resource', u'/tmp/test.c'), (u'name', u'test.c'), (u'version', u'1')]))]
+        expected_data = [dict(OrderedDict(
+            [(u'about_resource', u'/tmp/test.c'), (u'name', u'test.c'), (u'version', u'1')]))]
 
         assert len(data) == len(expected_data)
         for d in data:
@@ -84,22 +85,23 @@ def test_normalize_dict_data_scancode(self):
         json_data = read_json(test_file)
         data = normalize_dict_data(json_data)
         expected_data = [OrderedDict([(u'path', u'samples'),
-                                 (u'type', u'directory'),
-                                 (u'name', u'samples'),
-                                 (u'base_name', u'samples'),
-                                 (u'extension', u''), (u'size', 0),
-                                 (u'date', None), (u'sha1', None), (u'md5', None),
-                                 (u'mime_type', None), (u'file_type', None),
-                                 (u'programming_language', None),
-                                 (u'is_binary', False), (u'is_text', False),
-                                 (u'is_archive', False), (u'is_media', False),
-                                 (u'is_source', False), (u'is_script', False),
-                                 (u'licenses', []), (u'license_expressions', []),
-                                 (u'copyrights', []), (u'holders', []),
-                                 (u'authors', []), (u'packages', []),
-                                 (u'emails', []), (u'urls', []),
-                                 (u'files_count', 33), (u'dirs_count', 10),
-                                 (u'size_count', 1161083), (u'scan_errors', [])])]
+                                      (u'type', u'directory'),
+                                      (u'name', u'samples'),
+                                      (u'base_name', u'samples'),
+                                      (u'extension', u''), (u'size', 0),
+                                      (u'date', None), (u'sha1',
+                                                        None), (u'md5', None),
+                                      (u'mime_type', None), (u'file_type', None),
+                                      (u'programming_language', None),
+                                      (u'is_binary', False), (u'is_text', False),
+                                      (u'is_archive', False), (u'is_media', False),
+                                      (u'is_source', False), (u'is_script', False),
+                                      (u'licenses', []), (u'license_expressions', []),
+                                      (u'copyrights', []), (u'holders', []),
+                                      (u'authors', []), (u'packages', []),
+                                      (u'emails', []), (u'urls', []),
+                                      (u'files_count', 33), (u'dirs_count', 10),
+                                      (u'size_count', 1161083), (u'scan_errors', [])])]
         assert data == expected_data
 
     def test_normalize_dict_data_json(self):
@@ -116,19 +118,19 @@ def test_normalize_dict_data_json(self):
 
     def test_normalize_dict_data_json_array(self):
         json_data = [OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit/'),
-                    (u'Component', u'AboutCode-toolkit'),
-                    (u'version', u'1.0'), (u'temp', u'fpp')]),
-                    OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit1/'),
-                    (u'Component', u'AboutCode-toolkit1'),
-                    (u'version', u'1.1'), (u'temp', u'foo')])]
+                                  (u'Component', u'AboutCode-toolkit'),
+                                  (u'version', u'1.0'), (u'temp', u'fpp')]),
+                     OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit1/'),
+                                  (u'Component', u'AboutCode-toolkit1'),
+                                  (u'version', u'1.1'), (u'temp', u'foo')])]
         data = normalize_dict_data(json_data)
         expected_data = [OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit/'),
-                        (u'Component', u'AboutCode-toolkit'),
-                        (u'version', u'1.0'), (u'temp', u'fpp')]),
-                        OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit1/'),
-                        (u'Component', u'AboutCode-toolkit1'),
-                        (u'version', u'1.1'),
-                        (u'temp', u'foo')])]
+                                      (u'Component', u'AboutCode-toolkit'),
+                                      (u'version', u'1.0'), (u'temp', u'fpp')]),
+                         OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit1/'),
+                                      (u'Component', u'AboutCode-toolkit1'),
+                                      (u'version', u'1.1'),
+                                      (u'temp', u'foo')])]
         assert data == expected_data
 
     def test_check_duplicate_fields(self):
@@ -144,8 +146,10 @@ def test_strip_trailing_fields_csv(self):
         assert result == expected
 
     def test_strip_trailing_fields_json(self):
-        test = [OrderedDict([(u'about_resource', u'/this.c'), (u'name ', u'this.c'), (u' version ', u'0.11.0')])]
-        expected = [OrderedDict([(u'about_resource', u'/this.c'), (u'name', u'this.c'), (u'version', u'0.11.0')])]
+        test = [OrderedDict([(u'about_resource', u'/this.c'),
+                            (u'name ', u'this.c'), (u' version ', u'0.11.0')])]
+        expected = [OrderedDict(
+            [(u'about_resource', u'/this.c'), (u'name', u'this.c'), (u'version', u'0.11.0')])]
         result = strip_trailing_fields_json(test)
         assert result == expected
 
@@ -190,4 +194,66 @@ def test_transform_json(self):
                      'Component': 'AboutCode-toolkit',
                      'Confirmed Version': '123', 'notes': ''}]
         assert len(err) == 0
-        assert data == expected
+        assert data == expected
+
+    def test_apply_renamings(self):
+        data = [OrderedDict([(u'Directory/Filename', u'/tmp/test.c'),
+                             (u'Component', u'test.c'), (u'version', u'1'),
+                             (u'notes', u'test'), (u'temp', u'foo')])]
+        configuration = get_test_loc('test_transform/configuration')
+        transformer = Transformer.from_file(configuration)
+
+        expected = [OrderedDict([(u'about_resource', u'/tmp/test.c'), (u'name',
+                                 u'test.c'), (u'version', u'1'), (u'notes', u'test'), (u'temp', u'foo')])]
+        renamed_field_data = transformer.apply_renamings(data)
+        assert renamed_field_data == expected
+
+    def test_apply_renamings_nested_list(self):
+        data = [{'path': 'samples/JGroups-error.log', 'name': 'JGroups-error.log', 'license_detections': [{'license_expression': 'apache-1.1 AND apache-2.0', 'matches': [
+            {'score': 90.0, 'start_line': 4, 'end_line': 4, 'license_expression': 'apache-1.1'}, {'score': 100.0, 'start_line': 5, 'end_line': 5, 'license_expression': 'apache-2.0'}]}]}]
+        configuration = get_test_loc('test_transform/configuration3')
+        transformer = Transformer.from_file(configuration)
+
+        expected = [{'about_resource': 'samples/JGroups-error.log', 'name': 'JGroups-error.log', 'license_detections': [{'license_expression': 'apache-1.1 AND apache-2.0', 'matches': [
+            {'score_renamed': 90.0, 'start_line': 4, 'end_line': 4, 'license_expression': 'apache-1.1'}, {'score_renamed': 100.0, 'start_line': 5, 'end_line': 5, 'license_expression': 'apache-2.0'}]}]}]
+        updated_data = transformer.apply_renamings(data)
+        assert updated_data == expected
+
+    def test_filter_excluded(self):
+        data = [OrderedDict([(u'Directory/Filename', u'/tmp/test.c'),
+                             (u'Component', u'test.c'), (u'version', u'1'),
+                             (u'notes', u'test'), (u'temp', u'foo')])]
+        configuration = get_test_loc('test_transform/configuration')
+        transformer = Transformer.from_file(configuration)
+
+        expected = [OrderedDict([(u'Directory/Filename', u'/tmp/test.c'), (u'Component',
+                                 u'test.c'), (u'version', u'1'), (u'notes', u'test')])]
+        updated_data = transformer.filter_excluded(data)
+        assert updated_data == expected
+
+    def test_filter_excluded_nested_list(self):
+        data = [{'path': 'samples/JGroups-error.log', 'type': 'file', 'name': 'JGroups-error.log', 'license_detections': [{'license_expression': 'apache-1.1 AND apache-2.0', 'matches': [
+            {'score': 90.0, 'start_line': 4, 'end_line': 4, 'license_expression': 'apache-1.1'}, {'score': 100.0, 'start_line': 5, 'end_line': 5, 'license_expression': 'apache-2.0'}]}]}]
+        configuration = get_test_loc('test_transform/configuration3')
+        transformer = Transformer.from_file(configuration)
+
+        expected = [{'path': 'samples/JGroups-error.log', 'name': 'JGroups-error.log', 'license_detections': [{'license_expression': 'apache-1.1 AND apache-2.0', 'matches': [
+            {'score': 90.0, 'end_line': 4, 'license_expression': 'apache-1.1'}, {'score': 100.0, 'end_line': 5, 'license_expression': 'apache-2.0'}]}]}]
+        updated_data = transformer.filter_excluded(data)
+        assert updated_data == expected
+
+    def test_filter_fields(self):
+        data = [OrderedDict([(u'about_resource', u'/tmp/test.c'),
+                             (u'name', u'test.c'), (u'version', u'1'),
+                             (u'notes', u'test'), (u'temp', u'foo')])]
+        configuration = get_test_loc('test_transform/configuration')
+        transformer = Transformer.from_file(configuration)
+
+        updated_data = transformer.filter_fields(data)
+
+        expected = [OrderedDict([(u'about_resource', u'/tmp/test.c'),
+                                 (u'name', u'test.c'), (u'version', u'1'),
+                                 (u'temp', u'foo')])]
+
+        for d in updated_data:
+            assert dict(d) in expected

tests/testdata/test_transform/configuration3

@@ -0,0 +1,15 @@
+field_renamings:
+    about_resource : 'path'
+	score_renamed : score
+	size_renamed : size
+required_fields:
+	- about_resource
+	- name
+exclude_fields:
+	- sha1
+	- sha256
+	- md5
+	- type
+	- start_line
+	- matched_length
+	- scan_errors