fix errors in jsonToArray function

Author: jreadey

src/h5json/array_util.py

@@ -107,19 +107,6 @@ def jsonToArray(data_shape, data_dtype, data_json):
     """
     Return numpy array from the given json array.
     """
-    # print(f"jsonToArray: data_shape: {data_shape}, data_dtype: {data_dtype} data_json: {data_json}")
-    def fillVlenArray(rank, data, arr, index):
-        # print(f"fillVlenArray rank: {rank} data: {data} arr: {arr} index: {index}")
-        if arr.shape == ():
-            arr[()] = data
-        else:
-            for i in range(len(data)):
-                if rank > 1:
-                    index = fillVlenArray(rank - 1, data[i], arr, index)
-                else:
-                    arr[index] = data[i]
-                    index += 1
-        return index
 
     if data_json is None:
         return np.array([]).astype(data_dtype)
@@ -131,42 +118,30 @@ def fillVlenArray(rank, data, arr, index):
     # need some special conversion for compound types --
     # each element must be a tuple, but the JSON decoder
     # gives us a list instead.
-    if len(data_dtype) > 1 and not isinstance(data_json, (list, tuple)):
+    if len(data_dtype) > 0 and not isinstance(data_json, (list, tuple)):
         raise TypeError("expected list data for compound data type")
     npoints = getNumElements(data_shape)
     np_shape_rank = len(data_shape)
 
     if type(data_json) in (list, tuple):
-        converted_data = []
-        if np_shape_rank > 0 and npoints == 1 and len(data_json) == len(data_dtype):
-            converted_data.append(toTuple(0, data_json))
-        else:
-            converted_data = toTuple(np_shape_rank, data_json)
-        data_json = converted_data
+        data_json = toTuple(np_shape_rank, data_json)
 
-    if isVlen(data_dtype):
-        if np_shape_rank == 0 and npoints == 1:
-            arr_shape = ()
-        else:
-            arr_shape = (npoints,)
-        arr = np.zeros(arr_shape, dtype=data_dtype)
-        fillVlenArray(np_shape_rank, data_json, arr, 0)
-    else:
-        try:
-            arr = np.array(data_json, dtype=data_dtype)
-        except UnicodeEncodeError:
-            # Unable to encode data, encode as utf8 with surrogate escaping
-            data_json = toTuple(np_shape_rank, data_json, encoding="utf8")
-            arr = np.array(data_json, dtype=data_dtype)
+    arr = np.zeros(data_shape, dtype=data_dtype)
+
+    try:
+        # arr = np.array(data_json, dtype=data_dtype)
+        arr[...] = data_json
+    except UnicodeEncodeError:
+        # Unable to encode data, encode as utf8 with surrogate escaping
+        data_json = toTuple(np_shape_rank, data_json, encoding="utf8")
+        arr[...] = data_json
     # raise an exception of the array shape doesn't match the selection shape
     # allow if the array is a scalar and the selection shape is one element,
     # numpy is ok with this
     if arr.size != npoints:
         msg = "Input data doesn't match selection number of elements"
         msg += f" Expected {npoints}, but received: {arr.size}"
         raise ValueError(msg)
-    if arr.shape != data_shape:
-        arr = arr.reshape(data_shape)  # reshape to match selection
 
     return arr
 

test/unit/array_util_test.py

@@ -82,6 +82,27 @@ def testToTuple(self):
         out = toTuple(1, data3d)  # treat input a 1d array of compound type of compound types
         self.assertEqual([((0, 0.0), (1, 0.1)), ((2, 0.2), (3, 0.3))], out)
 
+    def testToTupleStrData(self):
+        data = "a string!"
+        out = toTuple(0, data)
+        self.assertEqual(data, out)
+
+        data = ["a string!"]
+        out = toTuple(1, data)
+        self.assertEqual(data, out)
+
+        data = ["a string2"]
+        out = toTuple(1, data)
+        self.assertEqual(data, out)
+
+        data = [["partA", "partB", "partC"],]
+        out = toTuple(1, data)
+        self.assertEqual([("partA", "partB", "partC"), ], out)
+
+        data = [[[4, 8, 12], "four"], [[5, 10, 15], "five"]]
+        out = toTuple(1, data)
+        self.assertEqual([((4, 8, 12), 'four'), ((5, 10, 15), 'five')], out)
+
     def testGetNumElements(self):
         shape = (4,)
         nelements = getNumElements(shape)
@@ -98,7 +119,6 @@ def testGetNumElements(self):
     def testJsonToArray(self):
 
         # simple integer
-
         dt = np.dtype("i4")
         shape = [4, ]
         data = [0, 2, 4, 6]
@@ -151,6 +171,14 @@ def testJsonToArray(self):
         val = out[0]
         self.assertEqual(val, data)
 
+        # VLEN multi element
+        shape = [5, ]
+        data = ["parting", "is", "such", "sweet", "sorrow"]
+        out = jsonToArray(shape, dt, data)
+        self.assertTrue(isinstance(out, np.ndarray))
+        self.assertEqual(out.shape, (5, ))
+        self.assertEqual(out[4], 'sorrow')
+
         # VLEN ascii
         dt = special_dtype(vlen=bytes)
         data = [b"one", b"two", b"three", b"four", b"five"]
@@ -167,22 +195,6 @@ def testJsonToArray(self):
         self.assertEqual(out[2], b"three")
         self.assertEqual(out[3], b"four")
 
-        # VLEN str
-        dt = special_dtype(vlen=str)
-        data = [
-            [b"part 1 - section A", b"part 1 - section B"],
-            [b"part 2 - section A", b"part 2 - section B"],
-        ]
-        shape = [2,]
-        out = jsonToArray(shape, dt, data)
-        self.assertTrue(isinstance(out, np.ndarray))
-        self.assertTrue("vlen" in out.dtype.metadata)
-        self.assertEqual(out.dtype.metadata["vlen"], str)
-        self.assertEqual(out.dtype.kind, "O")
-        self.assertEqual(out.shape, (2,))
-        self.assertEqual(out[0], tuple(data[0]))
-        self.assertEqual(out[1], tuple(data[1]))
-
         # VLEN unicode
         dt = special_dtype(vlen=bytes)
         data = ["one", "two", "three", "four", "five"]
@@ -207,6 +219,12 @@ def testJsonToArray(self):
         self.assertTrue(isinstance(out, np.ndarray))
         self.assertEqual(out[()], data)
 
+        data = ["I'm an UTF-8 null terminated string",]
+        shape = [1,]
+        out = jsonToArray(shape, dt, data)
+        self.assertTrue(isinstance(out, np.ndarray))
+        self.assertEqual(out[0], data[0])
+
         dt = np.dtype("S12")
         data = "eight: \u516b"
         out = jsonToArray(shape, dt, data)
@@ -223,9 +241,16 @@ def testJsonToArray(self):
         dt = np.dtype("S12")
         data = "eight: \u516b"
         out = jsonToArray(shape, dt, data)
+        self.assertTrue(isinstance(out, np.ndarray))
         self.assertEqual(out[0], b'eight: \xe5\x85\xab')
 
         # VLEN data
+        shape = []
+        dt = special_dtype(vlen=np.dtype("S10"))
+        data = ["foo", "bar"]
+        out = jsonToArray(shape, dt, data)
+        self.assertTrue(isinstance(out, np.ndarray))
+
         dt = special_dtype(vlen=np.dtype("int32"))
         shape = [4, ]
         data = [
@@ -321,7 +346,7 @@ def testJsonToArray(self):
         e1 = out[1].tolist()
         self.assertEqual(e1, (5, b"five"))
 
-        data = [6, "six"]
+        data = [[6, "six"],]
         shape = [1,]
         out = jsonToArray(shape, dt, data)
         self.assertTrue(isinstance(out, np.ndarray))
@@ -352,7 +377,7 @@ def testJsonToArray(self):
         self.assertEqual(e0, (4, "four"))
 
         shape = [1, ]
-        data = [6, "six"]
+        data = [[6, "six"],]
         out = jsonToArray(shape, dt, data)
         self.assertTrue(isinstance(out, np.ndarray))
         self.assertEqual(out.shape, (1,))