Merge pull request #995 from A-Baji/fix-mym-#86

Fix issue with reading 32 bit blobs

Author: dimitri-yatsenko

CHANGELOG.md

@@ -1,5 +1,8 @@
 ## Release notes
 
+### 0.13.4 -- TBA
+* Add - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995
+
 ### 0.13.3 -- Feb 9, 2022
 * Bugfix - Fix error in listing ancestors, descendants with part tables.
 * Bugfix - Fix Python 3.10 compatibility (#983) PR #972

datajoint/blob.py

@@ -44,6 +44,10 @@
 
 bypass_serialization = False  # runtime setting to bypass blob (en|de)code
 
+# runtime setting to read integers as 32-bit to read blobs created by the 32-bit
+# version of the mYm library for MATLAB
+use_32bit_dims = False
+
 
 def len_u64(obj):
     return np.uint64(len(obj)).tobytes()
@@ -96,7 +100,7 @@ def unpack(self, blob):
             pass  # assume uncompressed but could be unrecognized compression
         else:
             self._pos += len(prefix)
-            blob_size = self.read_value('uint64')
+            blob_size = self.read_value()
             blob = compression[prefix](self._blob[self._pos:])
             assert len(blob) == blob_size
             self._blob = blob
@@ -191,8 +195,8 @@ def pack_blob(self, obj):
         raise DataJointError("Packing object of type %s currently not supported!" % type(obj))
 
     def read_array(self):
-        n_dims = int(self.read_value('uint64'))
-        shape = self.read_value('uint64', count=n_dims)
+        n_dims = int(self.read_value())
+        shape = self.read_value(count=n_dims)
         n_elem = np.prod(shape, dtype=int)
         dtype_id, is_complex = self.read_value('uint32', 2)
         dtype = dtype_list[dtype_id]
@@ -365,7 +369,7 @@ def read_struct(self):
             return np.array(None)  # empty array
         field_names = [self.read_zero_terminated_string() for _ in range(n_fields)]
         raw_data = [
-            tuple(self.read_blob(n_bytes=int(self.read_value('uint64'))) for _ in range(n_fields))
+            tuple(self.read_blob(n_bytes=int(self.read_value())) for _ in range(n_fields))
             for __ in range(n_elem)]
         data = np.array(raw_data, dtype=list(zip(field_names, repeat(object))))
         return self.squeeze(data.reshape(shape, order="F"), convert_to_scalar=False).view(MatStruct)
@@ -431,7 +435,9 @@ def read_zero_terminated_string(self):
         self._pos = target + 1
         return data
 
-    def read_value(self, dtype='uint64', count=1):
+    def read_value(self, dtype=None, count=1):
+        if dtype is None:
+            dtype = 'uint32' if use_32bit_dims else 'uint64'
         data = np.frombuffer(self._blob, dtype=dtype, count=count, offset=self._pos)
         self._pos += data.dtype.itemsize * data.size
         return data[0] if count == 1 else data

datajoint/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.13.3"
+__version__ = "0.13.4"
 
 assert len(__version__) <= 10  # The log table limits version to the 10 characters

docs-parts/intro/Releases_lang1.rst

@@ -1,3 +1,7 @@
+0.13.4 -- TBA
+----------------------
+* Add - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995
+
 0.13.3 -- Feb 9, 2022
 ----------------------
 * Bugfix - Fix error in listing ancestors, descendants with part tables.

tests/schema.py

@@ -449,3 +449,12 @@ class Stimulus(dj.Lookup):
     contrast: int
     brightness: int
     """
+
+
+@schema
+class Longblob(dj.Manual):
+    definition = """
+    id: int
+    ---
+    data: longblob
+    """

tests/test_blob.py

@@ -1,5 +1,7 @@
+import datajoint as dj
 import numpy as np
 import uuid
+from . import schema
 from decimal import Decimal
 from datetime import datetime
 from datajoint.blob import pack, unpack
@@ -129,3 +131,41 @@ def test_complex():
 
     x = np.int16(np.random.randn(1, 2, 3)) + 1j*np.int16(np.random.randn(1, 2, 3))
     assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
+
+
+def test_insert_longblob():
+    insert_dj_blob = {'id': 1, 'data': [1, 2, 3]}
+    schema.Longblob.insert1(insert_dj_blob)
+    assert (schema.Longblob & 'id=1').fetch1() == insert_dj_blob
+    (schema.Longblob & 'id=1').delete()
+
+    query_mym_blob = {'id': 1, 'data': np.array([1, 2, 3])}
+    schema.Longblob.insert1(query_mym_blob)
+    assert (schema.Longblob & 'id=1').fetch1()['data'].all() == query_mym_blob['data'].all()
+    (schema.Longblob & 'id=1').delete()
+
+    query_32_blob = ("INSERT INTO djtest_test1.longblob (id, data) VALUES (1, "
+                "X'6D596D00530200000001000000010000000400000068697473007369646573007461736B73007374"
+                "616765004D000000410200000001000000070000000600000000000000000000000000F8FF00000000"
+                "0000F03F000000000000F03F0000000000000000000000000000F03F00000000000000000000000000"
+                "00F8FF230000004102000000010000000700000004000000000000006C006C006C006C00720072006C"
+                "0023000000410200000001000000070000000400000000000000640064006400640064006400640025"
+                "00000041020000000100000008000000040000000000000053007400610067006500200031003000')")
+    dj.conn().query(query_32_blob).fetchall()
+    dj.blob.use_32bit_dims = True
+    assert (schema.Longblob & 'id=1').fetch1() == {
+    'id': 1, 'data':
+        np.rec.array(
+            [[
+                (
+                    np.array([[np.nan,  1.,  1.,  0.,  1.,  0., np.nan]]),
+                    np.array(['llllrrl'], dtype='<U7'),
+                    np.array(['ddddddd'], dtype='<U7'),
+                    np.array(['Stage 10'], dtype='<U8')
+                )
+            ]],
+            dtype=[('hits', 'O'), ('sides', 'O'), ('tasks', 'O'), ('stage', 'O')]
+        )
+    }
+    (schema.Longblob & 'id=1').delete()
+    dj.blob.use_32bit_dims = False