[_707] support for subclassing iRODSMeta

Example: iRODSBinOrStringMeta allows storing arbitrary octet strings in metadata.

[_707] test of iRODSMeta subclass which stores bytestring value and/or unit

This required some more corrections and tweaks of the iRODSMeta_type handling.

[_707] iRODSMeta name could contain Unicode, ergo should not be included in translations

[_707] do remove properly

Author: d-w-moore

irods/manager/metadata_manager.py

@@ -30,7 +30,11 @@ class InvalidAtomicAVURequest(Exception):
 class MetadataManager(Manager):
 
     def __init__(self, *_):
-        self._opts = {'admin':False, 'timestamps':False}
+        self._opts = {
+            'admin':False,
+            'timestamps':False,
+            'iRODSMeta_type':iRODSMeta
+        }
         super().__init__(*_)
 
     @property
@@ -39,6 +43,7 @@ def use_timestamps(self):
 
     __kw : Dict[str, Any] = {}  # default (empty) keywords
 
+
     def _updated_keywords(self, opts):
         kw_ = self.__kw.copy()
         kw_.update(opts)
@@ -115,9 +120,9 @@ def meta_opts(row):
             return opts
 
         return [
-            iRODSMeta(
-                row[model.name], row[model.value], row[model.units], **meta_opts(row)
-            )
+            self._opts['iRODSMeta_type'](None,None,None)._from_column_triple(
+                row[model.name], row[model.value], row[model.units],
+                **meta_opts(row))
             for row in results
         ]
 
@@ -128,9 +133,7 @@ def add(self, model_cls, path, meta, **opts):
             "add",
             "-" + resource_type,
             path,
-            meta.name,
-            meta.value,
-            meta.units,
+            *meta._to_column_triple(),
             **self._updated_keywords(opts)
         )
         request = iRODSMessage(
@@ -147,9 +150,7 @@ def remove(self, model_cls, path, meta, **opts):
             "rm",
             "-" + resource_type,
             path,
-            meta.name,
-            meta.value,
-            meta.units,
+            *meta._to_column_triple(),
             **self._updated_keywords(opts)
         )
         request = iRODSMessage(
@@ -186,9 +187,7 @@ def set(self, model_cls, path, meta, **opts):
             "set",
             "-" + resource_type,
             path,
-            meta.name,
-            meta.value,
-            meta.units,
+            *meta._to_column_triple(),
             **self._updated_keywords(opts)
         )
         request = iRODSMessage(

irods/meta.py

@@ -1,14 +1,40 @@
+import base64
+import copy
+
+
 class iRODSMeta:
 
+    def _to_column_triple(self):
+        return (self.name ,self.FX(self.value)) + (('',) if not self.units else (self.FX(self.units),))
+
+    def _from_column_triple(self, name, value, units, **kw):
+        self.__low_level_init(name,
+                              self.RX(value),
+                              units=None if not units else self.RX(units),
+                              **kw)
+        return self
+
+    RX = FX = staticmethod(lambda _:_)
+    INIT_KW_ARGS = 'units avu_id create_time modify_time'.split()
+
     def __init__(
-        self, name, value, units=None, avu_id=None, create_time=None, modify_time=None
+        self, name, value, /, units=None, *, avu_id=None, create_time=None, modify_time=None,
     ):
-        self.avu_id = avu_id
+        # Defer initialization for iRODSMeta(attribute,value,...) if neither attribute nor value is True under
+        # a 'bool' transformation.  In so doing we streamline initialization for iRODSMeta (and any subclasses)
+        # for alternatively populating via _from_column_triple(...).
+        # This is the pathway for allowing user-defined encodings of the iRODSMeta (byte-)string AVU components.)
+        if name or value:
+            # Note: calling locals() inside the dict comprehension would not access variables in this frame.
+            local_vars = locals()
+            kw = {name:local_vars.get(name) for name in self.INIT_KW_ARGS}
+            self.__low_level_init(name, value, **kw)
+
+    def __low_level_init(self, name, value, **kw):
         self.name = name
         self.value = value
-        self.units = units
-        self.create_time = create_time
-        self.modify_time = modify_time
+        for attr in self.INIT_KW_ARGS:
+            setattr(self, attr, kw.get(attr))
 
     def __eq__(self, other):
         return tuple(self) == tuple(other)
@@ -20,7 +46,20 @@ def __iter__(self):
             yield self.units
 
     def __repr__(self):
-        return "<iRODSMeta {avu_id} {name} {value} {units}>".format(**vars(self))
+        return f"<{self.__class__.__name__} {self.avu_id} {self.name} {self.value} {self.units}>"
+
+    def __hash__(self):
+        return hash(tuple(self))
+
+class iRODSBinOrStringMeta(iRODSMeta):
+
+    @staticmethod
+    def RX(value):
+        return value if value[0] != '\\' else base64.decodebytes(value[1:].encode('utf8'))
+
+    @staticmethod
+    def FX(value):
+        return b'\\' + base64.encodebytes(value).strip() if isinstance(value,(bytes,bytearray)) else value
 
 
 class BadAVUOperationKeyword(Exception):
@@ -84,9 +123,6 @@ def __init__(self, operation, avu, **kw):
             setattr(self, atr, locals()[atr])
 
 
-import copy
-
-
 class iRODSMetaCollection:
 
     def __call__(self, **opts):
@@ -138,7 +174,7 @@ def get_one(self, key):
     def _get_meta(self, *args):
         if not len(args):
             raise ValueError("Must specify an iRODSMeta object or key, value, units)")
-        return args[0] if len(args) == 1 else iRODSMeta(*args)
+        return args[0] if len(args) == 1 else self._manager._opts['iRODSMeta_type'](*args)
 
     def apply_atomic_operations(self, *avu_ops):
         self._manager.apply_atomic_operations(self._model_cls, self._path, *avu_ops)

irods/test/meta_test.py

@@ -14,6 +14,7 @@
     AVUOperation,
     BadAVUOperationValue,
     BadAVUOperationKeyword,
+    iRODSBinOrStringMeta,
 )
 from irods.models import DataObject, Collection, Resource, CollectionMeta
 import irods.test.helpers as helpers
@@ -798,6 +799,35 @@ def test_xml_mode_addresses_odd_metadata_characters__issue_582(self):
                 # in use, with the "odd" characters being present in the metadata value.
                 del obj.metadata[attr_str]
 
+    def test_binary_avu_fields__issue_707(self):
+        meta_coll = self.obj.metadata(iRODSMeta_type=iRODSBinOrStringMeta)
+        illegal_unicode_sequence = '\u1000'.encode('utf8')[:2]
+        avu_name = 'issue709'
+        meta_coll.set(
+            avu_name,
+            (value:=b'value_'+illegal_unicode_sequence),
+            (units:=b'units_'+illegal_unicode_sequence)
+        )
+
+        self.assertEqual(
+            meta_coll.get_one(avu_name),
+            (avu_name, value, units)
+        )
+        meta_coll.add(*(new_avu:=iRODSMeta(avu_name, '\u1000', '\u1001')))
+        all_relevant_avus = meta_coll.get_all(avu_name)
+        self.assertIn(new_avu, all_relevant_avus)
+
+        # these could be in a separate test. TODO - make issue regarding hash and sets of iRODSMeta
+        all_avus = meta_coll.items()
+        self.assertLessEqual(
+            set(all_relevant_avus),
+            set(all_avus),
+        )
+        self.assertIn(
+            (avu_name,
+            (value:=b'value_'+illegal_unicode_sequence),
+            (units:=b'units_'+illegal_unicode_sequence)), all_relevant_avus)
+
     def test_cascading_changes_of_metadata_manager_options__issue_709(self):
         d = None
         get_option = lambda metacoll, key: metacoll._manager._opts[key]