Fix peaklist save to / load from hdf5 header size exceed 64KB bug.

Author: akialbz

dimspy/portals/hdf5_portal.py

@@ -10,7 +10,8 @@
 
 """
 
-import os, logging, h5py
+import os, sys, logging, zlib, h5py
+import cPickle as cp
 import numpy as np
 from ast import literal_eval
 from dimspy.models.peaklist_tags import PeakList_Tags
@@ -24,6 +25,16 @@ def _eval(v):
     except (ValueError, SyntaxError):
         return str(v)
 
+def _packMeta(v):
+    dv = cp.dumps(v)
+    # in case unicode chars, which are 4 times larger then str in python
+    return ['O', dv] if sys.getsizeof(dv) / 1000. < 16 else \
+           ['C', zlib.compress(dv)]
+
+def _unpackMeta(v):
+    dv = zlib.decompress(v[1]) if v[0] == 'C' else v[1]
+    return cp.loads(dv)
+
 
 # peaklists portals
 def save_peaklists_as_hdf5(pkls, filename):
@@ -56,7 +67,7 @@ def _savepkl(i, pkl):
 
         dset.attrs['flag_attrs'] = pkl.flag_attributes
         dset.attrs['tags'] = [(t if type(t) in (tuple, list) else ('None', t)) for t in pkl.tags.to_list()]
-        for k, v in pkl.metadata.items(): dset.attrs['metadata_' + k] = v
+        for k, v in pkl.metadata.items(): dset.attrs['metadata_' + k] = _packMeta(v)
 
     map(lambda x: _savepkl(*x), enumerate(pkls))
 
@@ -89,7 +100,7 @@ def _loadpkl(ID):
         if dn[0] != 'mz' or dn[1] != 'intensity':
             raise IOError('PANIC: HDF5 dataset matrix not in order')
         pkl = PeakList(ID, dm[0].astype(np.float64), dm[1].astype(np.float64),
-                       **{k[9:]: v for k,v in dset.attrs.items() if k.startswith('metadata_')})
+                       **{k[9:]: _unpackMeta(v) for k,v in dset.attrs.items() if k.startswith('metadata_')})
 
         for n, v, t in zip(dn[2:], dm[2:], dt[2:]):
             pkl.add_attribute(n, v, t, is_flag=(n in dset.attrs['flag_attrs']), flagged_only=False)