Merge pull request #422 from srthkdb/417_json_transform

420: added `exclude_fields` to transform

Author: chinyeungli

docs/UsingAboutCodetoDocumentYourSoftwareAssets.md

@@ -287,6 +287,19 @@ and "version" columns and no other column:
         - name
         - version
 
+* exclude_fields:
+An optional list of field names that should be excluded in the transformed CSV/JSON. If
+this list is provided, all the fields from the source CSV/JSON that should be excluded
+in the target CSV/JSON must be listed. Excluding standard or required fields will cause
+an error. If this list is not provided, all source CSV/JSON fields are kept in the
+transformed target CSV/JSON.
+
+For instance with this configuration the target CSV/JSON will not contain the "type"
+and "temp" fields:
+    exclude_fields:
+        - type
+        - temp
+
 
 ## <a name="RungentoGenerateAboutCodeToolkitFiles">Run gen to Generate AboutCode Toolkit Files</a>
 

etc/scripts/irc-notify.py

@@ -139,7 +139,7 @@ def appveyor_vars():
         response = line.split()
 
         if response[0] == 'PING':
-            irc_file.send('PONG {}\r\n'.format(reponse[1]).encode())
+            irc_file.send('PONG {}\r\n'.format(response[1]).encode())
 
         elif response[1] == '433':
             irc_sock.send('NICK {}\r\n'.format(irc_nick).encode())

src/attributecode/transform.py

@@ -109,6 +109,10 @@ def transform_csv(rows, transformer):
         data = list(transformer.filter_fields(data))
         field_names = [c for c in field_names if c in transformer.field_filters]
 
+    if transformer.exclude_fields:
+        data = list(transformer.filter_excluded(data))
+        field_names = [c for c in field_names if c not in transformer.exclude_fields]
+
     errors = transformer.check_required_fields(data)
 
     return field_names, data, errors
@@ -162,6 +166,11 @@ def process_json_keys(data, renamings, transformer):
         new_data = list(transformer.filter_fields(new_data))
     else:
         new_data = list(new_data)
+    
+    if transformer.exclude_fields:
+        new_data = list(transformer.filter_excluded(new_data))
+    else:
+        new_data = list(new_data)
 
     errors = transformer.check_required_fields(new_data)
     return new_data, errors
@@ -212,6 +221,19 @@ def process_json_keys(data, renamings, transformer):
     field_filters:
         - name
         - version
+
+* exclude_fields:
+An optional list of field names that should be excluded in the transformed CSV/JSON. If
+this list is provided, all the fields from the source CSV/JSON that should be excluded
+in the target CSV/JSON must be listed. Excluding standard or required fields will cause
+an error. If this list is not provided, all source CSV/JSON fields are kept in the
+transformed target CSV/JSON.
+
+For instance with this configuration the target CSV/JSON will not contain the "type"
+and "temp" fields:
+    exclude_fields:
+        - type
+        - temp
 '''
 
 
@@ -222,6 +244,7 @@ class Transformer(object):
     field_renamings = attr.attrib(default=attr.Factory(dict))
     required_fields = attr.attrib(default=attr.Factory(list))
     field_filters = attr.attrib(default=attr.Factory(list))
+    exclude_fields = attr.attrib(default=attr.Factory(list))
 
     # a list of all the standard fields from AboutCode toolkit
     standard_fields = attr.attrib(default=attr.Factory(list), init=False)
@@ -245,6 +268,7 @@ def default(cls):
             field_renamings={},
             required_fields=[],
             field_filters=[],
+            exclude_fields=[],
         )
 
     @classmethod
@@ -259,6 +283,7 @@ def from_file(cls, location):
             field_renamings=data.get('field_renamings', {}),
             required_fields=data.get('required_fields', []),
             field_filters=data.get('field_filters', []),
+            exclude_fields=data.get('exclude_fields', []),
         )
 
     def check_required_fields(self, data):
@@ -317,6 +342,17 @@ def filter_fields(self, data):
             items = ((k, v) for k, v in entry.items() if k in field_filters)
             yield OrderedDict(items)
 
+    def filter_excluded(self, data):
+        """
+        Yield transformed dicts from a `data` list of dicts excluding 
+        fields with names in the `exclude_fields`of this Transformer.
+        Return the data unchanged if no `exclude_fields` exists.
+        """
+        exclude_fields = set(self.clean_fields(self.exclude_fields))
+        for entry in data:
+            items = ((k, v) for k, v in entry.items() if k not in exclude_fields)
+            yield OrderedDict(items)
+
 
 def check_duplicate_fields(field_names):
     """

tests/testdata/test_cmd/help/about_transform_config_help.txt

@@ -44,3 +44,16 @@ and "version" fields and no other field:
         - name
         - version
 
+* exclude_fields:
+An optional list of field names that should be excluded in the transformed CSV/JSON. If
+this list is provided, all the fields from the source CSV/JSON that should be excluded
+in the target CSV/JSON must be listed. Excluding standard or required fields will cause
+an error. If this list is not provided, all source CSV/JSON fields are kept in the
+transformed target CSV/JSON.
+
+For instance with this configuration the target CSV/JSON will not contain the "type"
+and "temp" fields:
+    exclude_fields:
+        - type
+        - temp
+

tests/testdata/test_transform/configuration

@@ -5,6 +5,9 @@ field_filters:
     - about_resource
     - name
     - version
+    - temp
 required_fields:
 	- name
-	- version
+	- version
+exclude_fields:
+    - temp

tests/testdata/test_transform/configuration_scancode

@@ -5,7 +5,9 @@ field_filters:
 	- name
 	- new_extension
 	- about_resource
+	- type
 required_fields:
 	- name
-
+exclude_fields:
+	- type
 

tests/testdata/test_transform/input.csv

@@ -1,2 +1,2 @@
-Directory/Filename,Component,version,notes
-/tmp/test.c, test.c,1,test
+Directory/Filename,Component,version,notes,temp
+/tmp/test.c, test.c,1,test,foo

tests/testdata/test_transform/input.json

@@ -2,5 +2,6 @@
     "Directory/Filename": "/aboutcode-toolkit/",
 	"Component": "AboutCode-toolkit",
 	"version": "1.2.3",
-	"note": "test"
+	"note": "test",
+	"temp": "foo"
 }

tests/testdata/test_transform/input_as_array.json

@@ -2,11 +2,13 @@
   {
     "Directory/Filename": "/aboutcode-toolkit/", 
     "Component": "AboutCode-toolkit",
-	"version": "1.0"
+  "version": "1.0",
+  "temp": "fpp"
   },
   {
     "Directory/Filename": "/aboutcode-toolkit1/", 
     "Component": "AboutCode-toolkit1",
-	"version": "1.1"
+  "version": "1.1",
+  "temp": "foo"
   }  
 ]