fix: support sub-meanings for datastore v2.20.3 (#1014)

Author: daniel-sanche

google/cloud/ndb/key.py

@@ -716,13 +716,13 @@ def reference(self):
             >>> key = ndb.Key("Trampoline", 88, project="xy", database="wv", namespace="zt")
             >>> key.reference()
             app: "xy"
-            name_space: "zt"
             path {
               element {
                 type: "Trampoline"
                 id: 88
               }
             }
+            name_space: "zt"
             database_id: "wv"
             <BLANKLINE>
         """

google/cloud/ndb/model.py

@@ -2698,14 +2698,26 @@ def _from_datastore(self, ds_entity, value):
         Need to check the ds_entity for a compressed meaning that would
         indicate we are getting a compressed value.
         """
-        if self._name in ds_entity._meanings:
-            meaning = ds_entity._meanings[self._name][0]
-            if meaning == _MEANING_COMPRESSED and not self._compressed:
-                if self._repeated:
-                    for sub_value in value:
-                        sub_value.b_val = zlib.decompress(sub_value.b_val)
-                else:
-                    value.b_val = zlib.decompress(value.b_val)
+        if self._name in ds_entity._meanings and not self._compressed:
+            root_meaning = ds_entity._meanings[self._name][0]
+            sub_meanings = None
+            # meaning may be a tuple. Attempt unwrap
+            if isinstance(root_meaning, tuple):
+                root_meaning, sub_meanings = root_meaning
+            # decompress values if needed
+            if root_meaning == _MEANING_COMPRESSED and not self._repeated:
+                value.b_val = zlib.decompress(value.b_val)
+            elif root_meaning == _MEANING_COMPRESSED and self._repeated:
+                for sub_value in value:
+                    sub_value.b_val = zlib.decompress(sub_value.b_val)
+            elif isinstance(sub_meanings, list) and self._repeated:
+                for idx, sub_value in enumerate(value):
+                    try:
+                        if sub_meanings[idx] == _MEANING_COMPRESSED:
+                            sub_value.b_val = zlib.decompress(sub_value.b_val)
+                    except IndexError:
+                        # value list size exceeds sub_meanings list
+                        break
         return value
 
     def _db_set_compressed_meaning(self, p):

setup.py

@@ -41,7 +41,7 @@ def main():
         readme = readme_file.read()
     dependencies = [
         "google-api-core[grpc] >= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*",
-        "google-cloud-datastore >= 2.16.0, < 3.0.0dev",
+        "google-cloud-datastore >= 2.16.0, != 2.20.2, < 3.0.0dev",
         "protobuf >= 3.20.2, <6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
         "pymemcache >= 2.1.0, < 5.0.0dev",
         "pytz >= 2018.3",

tests/unit/test_model.py

@@ -1929,9 +1929,8 @@ class ThisKind(model.Model):
         ds_entity = model._entity_to_ds_entity(entity)
         assert ds_entity["foo"] == compressed_value
 
-    @staticmethod
     @pytest.mark.usefixtures("in_context")
-    def test__from_datastore_compressed_repeated_to_compressed():
+    def test__from_datastore_compressed_repeated_to_compressed(self):
         class ThisKind(model.Model):
             foo = model.BlobProperty(compressed=True, repeated=True)
 
@@ -1955,9 +1954,48 @@ class ThisKind(model.Model):
         ds_entity = model._entity_to_ds_entity(entity)
         assert ds_entity["foo"] == [compressed_value_one, compressed_value_two]
 
-    @staticmethod
+    @pytest.mark.skipif(
+        [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2],
+        reason="uses meanings semantics from datastore v2.20.2 and later",
+    )
+    @pytest.mark.parametrize(
+        "meaning",
+        [
+            (model._MEANING_COMPRESSED, None),  # set root meaning
+            (model._MEANING_COMPRESSED, []),
+            (model._MEANING_COMPRESSED, [1, 1]),
+            (None, [model._MEANING_COMPRESSED] * 2),  # set sub-meanings
+        ],
+    )
     @pytest.mark.usefixtures("in_context")
-    def test__from_datastore_compressed_repeated_to_uncompressed():
+    def test__from_datastore_compressed_repeated_to_compressed_tuple_meaning(
+        self, meaning
+    ):
+        class ThisKind(model.Model):
+            foo = model.BlobProperty(compressed=True, repeated=True)
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        uncompressed_value_one = b"abc" * 1000
+        compressed_value_one = zlib.compress(uncompressed_value_one)
+        uncompressed_value_two = b"xyz" * 1000
+        compressed_value_two = zlib.compress(uncompressed_value_two)
+        compressed_value = [compressed_value_one, compressed_value_two]
+        datastore_entity.update({"foo": compressed_value})
+        meanings = {
+            "foo": (
+                meaning,
+                compressed_value,
+            )
+        }
+        datastore_entity._meanings = meanings
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        ds_entity = model._entity_to_ds_entity(entity)
+        assert ds_entity["foo"] == [compressed_value_one, compressed_value_two]
+
+    @pytest.mark.usefixtures("in_context")
+    def test__from_datastore_compressed_repeated_to_uncompressed(self):
         class ThisKind(model.Model):
             foo = model.BlobProperty(compressed=False, repeated=True)
 
@@ -1981,6 +2019,170 @@ class ThisKind(model.Model):
         ds_entity = model._entity_to_ds_entity(entity)
         assert ds_entity["foo"] == [uncompressed_value_one, uncompressed_value_two]
 
+    @pytest.mark.skipif(
+        [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2],
+        reason="uses meanings semantics from datastore v2.20.2 and later",
+    )
+    @pytest.mark.parametrize(
+        "meaning",
+        [
+            (model._MEANING_COMPRESSED, None),  # set root meaning
+            (model._MEANING_COMPRESSED, []),
+            (model._MEANING_COMPRESSED, [1, 1]),
+            (None, [model._MEANING_COMPRESSED] * 2),  # set sub-meanings
+        ],
+    )
+    @pytest.mark.usefixtures("in_context")
+    def test__from_datastore_compressed_repeated_to_uncompressed_tuple_meaning(
+        self, meaning
+    ):
+        class ThisKind(model.Model):
+            foo = model.BlobProperty(compressed=False, repeated=True)
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        uncompressed_value_one = b"abc" * 1000
+        compressed_value_one = zlib.compress(uncompressed_value_one)
+        uncompressed_value_two = b"xyz" * 1000
+        compressed_value_two = zlib.compress(uncompressed_value_two)
+        compressed_value = [compressed_value_one, compressed_value_two]
+        datastore_entity.update({"foo": compressed_value})
+        meanings = {
+            "foo": (
+                meaning,
+                compressed_value,
+            )
+        }
+        datastore_entity._meanings = meanings
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        ds_entity = model._entity_to_ds_entity(entity)
+        assert ds_entity["foo"] == [uncompressed_value_one, uncompressed_value_two]
+
+    @pytest.mark.skipif(
+        [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2],
+        reason="uses meanings semantics from datastore v2.20.2 and later",
+    )
+    @pytest.mark.parametrize(
+        "meaning",
+        [
+            (None, [model._MEANING_COMPRESSED, None]),
+            (None, [model._MEANING_COMPRESSED, None, None]),
+            (1, [model._MEANING_COMPRESSED, 1]),
+            (None, [model._MEANING_COMPRESSED]),
+        ],
+    )
+    @pytest.mark.usefixtures("in_context")
+    def test__from_datastore_compressed_repeated_to_uncompressed_mixed_meaning(
+        self, meaning
+    ):
+        """
+        One item is compressed, one uncompressed
+        """
+
+        class ThisKind(model.Model):
+            foo = model.BlobProperty(compressed=False, repeated=True)
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        uncompressed_value_one = b"abc" * 1000
+        compressed_value_one = zlib.compress(uncompressed_value_one)
+        uncompressed_value_two = b"xyz" * 1000
+        compressed_value_two = zlib.compress(uncompressed_value_two)
+        compressed_value = [compressed_value_one, compressed_value_two]
+        datastore_entity.update({"foo": compressed_value})
+        meanings = {
+            "foo": (
+                meaning,
+                compressed_value,
+            )
+        }
+        datastore_entity._meanings = meanings
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        ds_entity = model._entity_to_ds_entity(entity)
+        assert ds_entity["foo"] == [uncompressed_value_one, compressed_value_two]
+
+    @pytest.mark.skipif(
+        [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2],
+        reason="uses meanings semantics from datastore v2.20.2 and later",
+    )
+    @pytest.mark.parametrize(
+        "meaning",
+        [
+            (None, None),
+            (None, []),
+            (None, [None]),
+            (None, [None, None]),
+            (1, []),
+            (1, [1]),
+            (1, [1, 1]),
+        ],
+    )
+    @pytest.mark.usefixtures("in_context")
+    def test__from_datastore_compressed_repeated_no_meaning(self, meaning):
+        """
+        could be uncompressed, but meaning not set
+        """
+
+        class ThisKind(model.Model):
+            foo = model.BlobProperty(compressed=False, repeated=True)
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        uncompressed_value_one = b"abc" * 1000
+        compressed_value_one = zlib.compress(uncompressed_value_one)
+        uncompressed_value_two = b"xyz" * 1000
+        compressed_value_two = zlib.compress(uncompressed_value_two)
+        compressed_value = [compressed_value_one, compressed_value_two]
+        datastore_entity.update({"foo": compressed_value})
+        meanings = {
+            "foo": (
+                meaning,
+                compressed_value,
+            )
+        }
+        datastore_entity._meanings = meanings
+        protobuf = helpers.entity_to_protobuf(datastore_entity)
+        entity = model._entity_from_protobuf(protobuf)
+        ds_entity = model._entity_to_ds_entity(entity)
+        assert ds_entity["foo"] == [compressed_value_one, compressed_value_two]
+
+    @staticmethod
+    @pytest.mark.usefixtures("in_context")
+    def test__from_datastore_large_value_list():
+        """
+        try calling _from_datastore with a meaning list smaller than the value list
+        """
+
+        prop = model.BlobProperty(compressed=False, repeated=True, name="foo")
+
+        key = datastore.Key("ThisKind", 123, project="testing")
+        datastore_entity = datastore.Entity(key=key)
+        uncompressed_value_one = b"abc" * 1000
+        compressed_value_one = zlib.compress(uncompressed_value_one)
+        uncompressed_value_two = b"xyz" * 1000
+        compressed_value_two = zlib.compress(uncompressed_value_two)
+        compressed_value = [
+            model._BaseValue(compressed_value_one),
+            model._BaseValue(compressed_value_two),
+        ]
+        datastore_entity.update({"foo": compressed_value})
+        meanings = {
+            "foo": (
+                (None, [model._MEANING_COMPRESSED]),
+                compressed_value,
+            )
+        }
+
+        datastore_entity._meanings = meanings
+
+        updated_value = prop._from_datastore(datastore_entity, compressed_value)
+        assert len(updated_value) == 2
+        assert updated_value[0].b_val == uncompressed_value_one
+        # second value should remain compressed
+        assert updated_value[1].b_val == compressed_value_two
+
     @staticmethod
     @pytest.mark.usefixtures("in_context")
     def test__from_datastore_uncompressed_to_uncompressed():