Merge pull request #401 from OpenDataServices/flatten-array-of-arrays

json_input: Add support for flattening an array of arrays

Author: James (ODSC)

CHANGELOG.md

@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Added
+
+## [0.18.0] - 2022-09-26
+
+- Add support for flattening an array of arrays https://github.com/OpenDataServices/flatten-tool/issues/398
+
 ## [0.17.2] - 2022-06-15
 
 ### Fixed

flattentool/json_input.py

@@ -419,10 +419,20 @@ def parse_json_dict(
                     # Check for an array of BASIC types
                     # TODO Make this check the schema
                     # TODO Error if the any of the values contain the separator
-                    # TODO Support doubly nested arrays
                     flattened_dict[sheet_key(sheet, parent_name + key)] = ";".join(
                         map(str, value)
                     )
+                # Arrays of arrays
+                elif all(
+                    l not in BASIC_TYPES
+                    and not hasattr(l, "items")
+                    and hasattr(l, "__iter__")
+                    and all(type(x) in BASIC_TYPES for x in l)
+                    for l in value
+                ):
+                    flattened_dict[sheet_key(sheet, parent_name + key)] = ";".join(
+                        map(lambda l: ",".join(map(str, l)), value)
+                    )
                 else:
                     if (
                         self.rollup and parent_name == ""

flattentool/tests/fixtures/360-giving-schema.json

@@ -623,6 +623,13 @@
       "description": "A web link pointing to the source of this data. This may be an original 360Giving data file, a file from which the data was converted, or an organisation website.",
       "weight": 25,
       "title": "Data Source"
+    },
+    "test_array_of_arrays": {
+        "type": "array",
+        "items": {
+            "type": "array",
+            "items": {"type": "string"}
+        }
     }
   }
 }

flattentool/tests/fixtures/fundingproviders-grants_fixed_2_grants.json

@@ -35,7 +35,8 @@
             ],
             "Grant type": "Large Awards",
             "Full name of applicant": "Miss Jane Roe",
-            "awardDate": "24/07/2014"
+            "awardDate": "24/07/2014",
+            "test_array_of_arrays": [["a", "r"], ["s", "t", "n"], ["e"]]
         },
         {
             "recipientOrganization": [

flattentool/tests/test_json_input.py

@@ -152,6 +152,17 @@ def test_parse_array():
     assert parser.sub_sheets == {}
 
 
+def test_parse_array_of_arrays():
+    parser = JSONParser(
+        root_json_dict=[
+            OrderedDict([("testarray", [["item", "anotheritem", 42], ["a", "b", 1]])])
+        ]
+    )
+    assert list(parser.main_sheet) == ["testarray"]
+    assert list(parser.main_sheet.lines) == [{"testarray": "item,anotheritem,42;a,b,1"}]
+    assert parser.sub_sheets == {}
+
+
 def test_root_list_path():
     parser = JSONParser(
         root_json_dict={

setup.py

@@ -43,7 +43,7 @@ def run(self):
 
 setup(
     name="flattentool",
-    version="0.17.2",
+    version="0.18.0",
     author="Open Data Services",
     author_email="[email protected]",
     packages=["flattentool"],