From 98b6f40d8218f0cefe84e34321b6b9e1304f0bf9 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 18 Mar 2025 15:57:22 -0700 Subject: [PATCH 01/13] support tuple meaning fields --- google/cloud/ndb/model.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/google/cloud/ndb/model.py b/google/cloud/ndb/model.py index acfd10a8..bb4f3500 100644 --- a/google/cloud/ndb/model.py +++ b/google/cloud/ndb/model.py @@ -2698,14 +2698,26 @@ def _from_datastore(self, ds_entity, value): Need to check the ds_entity for a compressed meaning that would indicate we are getting a compressed value. """ - if self._name in ds_entity._meanings: - meaning = ds_entity._meanings[self._name][0] - if meaning == _MEANING_COMPRESSED and not self._compressed: - if self._repeated: - for sub_value in value: + if self._name in ds_entity._meanings and not self._compressed: + root_meaning = ds_entity._meanings[self._name][0] + sub_meanings = None + # meaning may be a tuple. Attempt unwrap + if isinstance(root_meaning, tuple): + root_meaning, sub_meanings = root_meaning + # decompress values if needed + if root_meaning == _MEANING_COMPRESSED and not self._repeated: + value.b_val = zlib.decompress(value.b_val) + elif root_meaning == _MEANING_COMPRESSED and self._repeated: + for sub_value in value: + sub_value.b_val = zlib.decompress(sub_value.b_val) + elif sub_meanings and self._repeated: + for idx, sub_value in enumerate(value): + try: + sub_value_meaning = sub_meanings[idx] + except IndexError: + sub_value_meaning = None + if sub_value_meaning == _MEANING_COMPRESSED: sub_value.b_val = zlib.decompress(sub_value.b_val) - else: - value.b_val = zlib.decompress(value.b_val) return value def _db_set_compressed_meaning(self, p): From 6769ead433a2b8ac037b71ca02072a72bb7f7b10 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 18 Mar 2025 16:16:49 -0700 Subject: [PATCH 02/13] fixed tests --- google/cloud/ndb/model.py | 2 +- tests/unit/test_model.py | 92 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/google/cloud/ndb/model.py b/google/cloud/ndb/model.py index bb4f3500..5e675fa7 100644 --- a/google/cloud/ndb/model.py +++ b/google/cloud/ndb/model.py @@ -2714,7 +2714,7 @@ def _from_datastore(self, ds_entity, value): for idx, sub_value in enumerate(value): try: sub_value_meaning = sub_meanings[idx] - except IndexError: + except IndexError: sub_value_meaning = None if sub_value_meaning == _MEANING_COMPRESSED: sub_value.b_val = zlib.decompress(sub_value.b_val) diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 14f03cef..3586b0ec 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -1929,9 +1929,13 @@ class ThisKind(model.Model): ds_entity = model._entity_to_ds_entity(entity) assert ds_entity["foo"] == compressed_value + @pytest.mark.skipif( + [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], + reason="uses meanings semantics from datastore v2.20.1 and lower", + ) @staticmethod @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_compressed(): + def test__from_datastore_compressed_repeated_to_compressed_legacy(): class ThisKind(model.Model): foo = model.BlobProperty(compressed=True, repeated=True) @@ -1955,9 +1959,53 @@ class ThisKind(model.Model): ds_entity = model._entity_to_ds_entity(entity) assert ds_entity["foo"] == [compressed_value_one, compressed_value_two] + @pytest.mark.skipif( + [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2], + reason="uses meanings semantics from datastore v2.20.2 and later", + ) + @pytest.mark.parametrize( + "meaning", + [ + (model._MEANING_COMPRESSED, None), # set root meaning + ( + None, + [model._MEANING_COMPRESSED, model._MEANING_COMPRESSED], + ), # set sub-meanings + ], + ) @staticmethod @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_uncompressed(): + def test__from_datastore_compressed_repeated_to_compressed(meaning): + class ThisKind(model.Model): + foo = model.BlobProperty(compressed=True, repeated=True) + + key = datastore.Key("ThisKind", 123, project="testing") + datastore_entity = datastore.Entity(key=key) + uncompressed_value_one = b"abc" * 1000 + compressed_value_one = zlib.compress(uncompressed_value_one) + uncompressed_value_two = b"xyz" * 1000 + compressed_value_two = zlib.compress(uncompressed_value_two) + compressed_value = [compressed_value_one, compressed_value_two] + datastore_entity.update({"foo": compressed_value}) + meanings = { + "foo": ( + meaning, + compressed_value, + ) + } + datastore_entity._meanings = meanings + protobuf = helpers.entity_to_protobuf(datastore_entity) + entity = model._entity_from_protobuf(protobuf) + ds_entity = model._entity_to_ds_entity(entity) + assert ds_entity["foo"] == [compressed_value_one, compressed_value_two] + + @pytest.mark.skipif( + [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], + reason="uses meanings semantics from datastore v2.20.1 and lower", + ) + @staticmethod + @pytest.mark.usefixtures("in_context") + def test__from_datastore_compressed_repeated_to_uncompressed_legacy(meaning): class ThisKind(model.Model): foo = model.BlobProperty(compressed=False, repeated=True) @@ -1981,6 +2029,46 @@ class ThisKind(model.Model): ds_entity = model._entity_to_ds_entity(entity) assert ds_entity["foo"] == [uncompressed_value_one, uncompressed_value_two] + @pytest.mark.skipif( + [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2], + reason="uses meanings semantics from datastore v2.20.2 and later", + ) + @pytest.mark.parametrize( + "meaning", + [ + (model._MEANING_COMPRESSED, None), # set root meaning + ( + None, + [model._MEANING_COMPRESSED, model._MEANING_COMPRESSED], + ), # set sub-meanings + ], + ) + @staticmethod + @pytest.mark.usefixtures("in_context") + def test__from_datastore_compressed_repeated_to_uncompressed(meaning): + class ThisKind(model.Model): + foo = model.BlobProperty(compressed=False, repeated=True) + + key = datastore.Key("ThisKind", 123, project="testing") + datastore_entity = datastore.Entity(key=key) + uncompressed_value_one = b"abc" * 1000 + compressed_value_one = zlib.compress(uncompressed_value_one) + uncompressed_value_two = b"xyz" * 1000 + compressed_value_two = zlib.compress(uncompressed_value_two) + compressed_value = [compressed_value_one, compressed_value_two] + datastore_entity.update({"foo": compressed_value}) + meanings = { + "foo": ( + meaning, + compressed_value, + ) + } + datastore_entity._meanings = meanings + protobuf = helpers.entity_to_protobuf(datastore_entity) + entity = model._entity_from_protobuf(protobuf) + ds_entity = model._entity_to_ds_entity(entity) + assert ds_entity["foo"] == [uncompressed_value_one, uncompressed_value_two] + @staticmethod @pytest.mark.usefixtures("in_context") def test__from_datastore_uncompressed_to_uncompressed(): From 91c3f16deb37aa2314a234024c25ec21d69c2996 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 18 Mar 2025 16:51:17 -0700 Subject: [PATCH 03/13] added test with mixed meanings --- tests/unit/test_model.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 3586b0ec..1032f032 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -2069,6 +2069,39 @@ class ThisKind(model.Model): ds_entity = model._entity_to_ds_entity(entity) assert ds_entity["foo"] == [uncompressed_value_one, uncompressed_value_two] + @pytest.mark.skipif( + [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2], + reason="uses meanings semantics from datastore v2.20.2 and later" + ) + @staticmethod + @pytest.mark.usefixtures("in_context") + def test__from_datastore_compressed_repeated_to_uncompressed_mixed_meaning(): + """ + One item is compressed, one uncompressed + """ + class ThisKind(model.Model): + foo = model.BlobProperty(compressed=False, repeated=True) + + key = datastore.Key("ThisKind", 123, project="testing") + datastore_entity = datastore.Entity(key=key) + uncompressed_value_one = b"abc" * 1000 + compressed_value_one = zlib.compress(uncompressed_value_one) + uncompressed_value_two = b"xyz" * 1000 + compressed_value_two = zlib.compress(uncompressed_value_two) + compressed_value = [compressed_value_one, compressed_value_two] + datastore_entity.update({"foo": compressed_value}) + meanings = { + "foo": ( + (None, [model._MEANING_COMPRESSED, None]), + compressed_value, + ) + } + datastore_entity._meanings = meanings + protobuf = helpers.entity_to_protobuf(datastore_entity) + entity = model._entity_from_protobuf(protobuf) + ds_entity = model._entity_to_ds_entity(entity) + assert ds_entity["foo"] == [uncompressed_value_one, compressed_value_two] + @staticmethod @pytest.mark.usefixtures("in_context") def test__from_datastore_uncompressed_to_uncompressed(): From 776d6a0da0ee49d7cf850ee3e8fb269083f4c336 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 18 Mar 2025 17:01:23 -0700 Subject: [PATCH 04/13] fixed tests that weren't running --- tests/unit/test_model.py | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 1032f032..8b086dea 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -1930,12 +1930,11 @@ class ThisKind(model.Model): assert ds_entity["foo"] == compressed_value @pytest.mark.skipif( - [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], + [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2], reason="uses meanings semantics from datastore v2.20.1 and lower", ) - @staticmethod @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_compressed_legacy(): + def test__from_datastore_compressed_repeated_to_compressed_legacy(self): class ThisKind(model.Model): foo = model.BlobProperty(compressed=True, repeated=True) @@ -1960,22 +1959,18 @@ class ThisKind(model.Model): assert ds_entity["foo"] == [compressed_value_one, compressed_value_two] @pytest.mark.skipif( - [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2], + [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], reason="uses meanings semantics from datastore v2.20.2 and later", ) @pytest.mark.parametrize( "meaning", [ (model._MEANING_COMPRESSED, None), # set root meaning - ( - None, - [model._MEANING_COMPRESSED, model._MEANING_COMPRESSED], - ), # set sub-meanings + (None, [model._MEANING_COMPRESSED] * 2), # set sub-meanings ], ) - @staticmethod @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_compressed(meaning): + def test__from_datastore_compressed_repeated_to_compressed(self, meaning): class ThisKind(model.Model): foo = model.BlobProperty(compressed=True, repeated=True) @@ -2000,12 +1995,11 @@ class ThisKind(model.Model): assert ds_entity["foo"] == [compressed_value_one, compressed_value_two] @pytest.mark.skipif( - [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], + [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2], reason="uses meanings semantics from datastore v2.20.1 and lower", ) - @staticmethod @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_uncompressed_legacy(meaning): + def test__from_datastore_compressed_repeated_to_uncompressed_legacy(self): class ThisKind(model.Model): foo = model.BlobProperty(compressed=False, repeated=True) @@ -2030,22 +2024,18 @@ class ThisKind(model.Model): assert ds_entity["foo"] == [uncompressed_value_one, uncompressed_value_two] @pytest.mark.skipif( - [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2], + [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], reason="uses meanings semantics from datastore v2.20.2 and later", ) @pytest.mark.parametrize( "meaning", [ (model._MEANING_COMPRESSED, None), # set root meaning - ( - None, - [model._MEANING_COMPRESSED, model._MEANING_COMPRESSED], - ), # set sub-meanings + (None, [model._MEANING_COMPRESSED] * 2), # set sub-meanings ], ) - @staticmethod @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_uncompressed(meaning): + def test__from_datastore_compressed_repeated_to_uncompressed(self, meaning): class ThisKind(model.Model): foo = model.BlobProperty(compressed=False, repeated=True) @@ -2070,15 +2060,15 @@ class ThisKind(model.Model): assert ds_entity["foo"] == [uncompressed_value_one, uncompressed_value_two] @pytest.mark.skipif( - [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2], - reason="uses meanings semantics from datastore v2.20.2 and later" + [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], + reason="uses meanings semantics from datastore v2.20.2 and later", ) - @staticmethod @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_uncompressed_mixed_meaning(): + def test__from_datastore_compressed_repeated_to_uncompressed_mixed_meaning(self): """ One item is compressed, one uncompressed """ + class ThisKind(model.Model): foo = model.BlobProperty(compressed=False, repeated=True) From 7fa0858e7107907c4db4185b37c68637cbb39d86 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 20 Mar 2025 15:27:08 -0700 Subject: [PATCH 05/13] cleaned up sub_meanings iteration --- google/cloud/ndb/model.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/google/cloud/ndb/model.py b/google/cloud/ndb/model.py index 5e675fa7..53d892c8 100644 --- a/google/cloud/ndb/model.py +++ b/google/cloud/ndb/model.py @@ -2710,14 +2710,14 @@ def _from_datastore(self, ds_entity, value): elif root_meaning == _MEANING_COMPRESSED and self._repeated: for sub_value in value: sub_value.b_val = zlib.decompress(sub_value.b_val) - elif sub_meanings and self._repeated: + elif sub_meanings and isinstance(sub_meanings, list) and self._repeated: for idx, sub_value in enumerate(value): try: - sub_value_meaning = sub_meanings[idx] + if sub_meanings[idx] == _MEANING_COMPRESSED: + sub_value.b_val = zlib.decompress(sub_value.b_val) except IndexError: - sub_value_meaning = None - if sub_value_meaning == _MEANING_COMPRESSED: - sub_value.b_val = zlib.decompress(sub_value.b_val) + # value list size exceeds sub_meanings list + break return value def _db_set_compressed_meaning(self, p): From 105caaa2ebe93f9058d91cd1cf718bb5d32d9ce9 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 20 Mar 2025 15:28:06 -0700 Subject: [PATCH 06/13] added tests --- tests/unit/test_model.py | 64 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 8b086dea..415ac143 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -1966,6 +1966,8 @@ class ThisKind(model.Model): "meaning", [ (model._MEANING_COMPRESSED, None), # set root meaning + (model._MEANING_COMPRESSED, []), + (model._MEANING_COMPRESSED, [1, 1]), (None, [model._MEANING_COMPRESSED] * 2), # set sub-meanings ], ) @@ -2031,6 +2033,8 @@ class ThisKind(model.Model): "meaning", [ (model._MEANING_COMPRESSED, None), # set root meaning + (model._MEANING_COMPRESSED, []), + (model._MEANING_COMPRESSED, [1, 1]), (None, [model._MEANING_COMPRESSED] * 2), # set sub-meanings ], ) @@ -2063,8 +2067,19 @@ class ThisKind(model.Model): [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], reason="uses meanings semantics from datastore v2.20.2 and later", ) + @pytest.mark.parametrize( + "meaning", + [ + (None, [model._MEANING_COMPRESSED, None]), + (None, [model._MEANING_COMPRESSED, None, None]), + (1, [model._MEANING_COMPRESSED, 1]), + (None, [model._MEANING_COMPRESSED]), + ], + ) @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_uncompressed_mixed_meaning(self): + def test__from_datastore_compressed_repeated_to_uncompressed_mixed_meaning( + self, meaning + ): """ One item is compressed, one uncompressed """ @@ -2082,7 +2097,7 @@ class ThisKind(model.Model): datastore_entity.update({"foo": compressed_value}) meanings = { "foo": ( - (None, [model._MEANING_COMPRESSED, None]), + meaning, compressed_value, ) } @@ -2092,6 +2107,51 @@ class ThisKind(model.Model): ds_entity = model._entity_to_ds_entity(entity) assert ds_entity["foo"] == [uncompressed_value_one, compressed_value_two] + @pytest.mark.skipif( + [int(v) for v in datastore.__version__.split(".")] < [2, 20, 2], + reason="uses meanings semantics from datastore v2.20.2 and later", + ) + @pytest.mark.parametrize( + "meaning", + [ + (None, None), + (None, []), + (None, [None]), + (None, [None, None]), + (1, []), + (1, [1]), + (1, [1, 1]), + ], + ) + @pytest.mark.usefixtures("in_context") + def test__from_datastore_compressed_repeated_no_meaning(self, meaning): + """ + could be uncompressed, but meaning not set + """ + + class ThisKind(model.Model): + foo = model.BlobProperty(compressed=False, repeated=True) + + key = datastore.Key("ThisKind", 123, project="testing") + datastore_entity = datastore.Entity(key=key) + uncompressed_value_one = b"abc" * 1000 + compressed_value_one = zlib.compress(uncompressed_value_one) + uncompressed_value_two = b"xyz" * 1000 + compressed_value_two = zlib.compress(uncompressed_value_two) + compressed_value = [compressed_value_one, compressed_value_two] + datastore_entity.update({"foo": compressed_value}) + meanings = { + "foo": ( + meaning, + compressed_value, + ) + } + datastore_entity._meanings = meanings + protobuf = helpers.entity_to_protobuf(datastore_entity) + entity = model._entity_from_protobuf(protobuf) + ds_entity = model._entity_to_ds_entity(entity) + assert ds_entity["foo"] == [compressed_value_one, compressed_value_two] + @staticmethod @pytest.mark.usefixtures("in_context") def test__from_datastore_uncompressed_to_uncompressed(): From 48ca292185adb3b1ecd8ca484ad914a9acd2064b Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 20 Mar 2025 15:30:58 -0700 Subject: [PATCH 07/13] removed redundant check --- google/cloud/ndb/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/ndb/model.py b/google/cloud/ndb/model.py index 53d892c8..c4d3cdb6 100644 --- a/google/cloud/ndb/model.py +++ b/google/cloud/ndb/model.py @@ -2710,7 +2710,7 @@ def _from_datastore(self, ds_entity, value): elif root_meaning == _MEANING_COMPRESSED and self._repeated: for sub_value in value: sub_value.b_val = zlib.decompress(sub_value.b_val) - elif sub_meanings and isinstance(sub_meanings, list) and self._repeated: + elif isinstance(sub_meanings, list) and self._repeated: for idx, sub_value in enumerate(value): try: if sub_meanings[idx] == _MEANING_COMPRESSED: From 0cd2259b20b7970ab77b7bc00c899ca8a0f1f3c0 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 21 Mar 2025 14:38:14 -0700 Subject: [PATCH 08/13] disallow datastore v2.20.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2bee63fb..fd6e94e2 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def main(): readme = readme_file.read() dependencies = [ "google-api-core[grpc] >= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", - "google-cloud-datastore >= 2.16.0, < 3.0.0dev", + "google-cloud-datastore >= 2.16.0, != 2.20.2, < 3.0.0dev", "protobuf >= 3.20.2, <6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", "pymemcache >= 2.1.0, < 5.0.0dev", "pytz >= 2018.3", From fd7c2f538cf92e5e84f8e14e58b54a41c0081128 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 21 Mar 2025 14:50:55 -0700 Subject: [PATCH 09/13] run legacy tests against latest version --- tests/unit/test_model.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 415ac143..0e17d6fb 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -1929,12 +1929,8 @@ class ThisKind(model.Model): ds_entity = model._entity_to_ds_entity(entity) assert ds_entity["foo"] == compressed_value - @pytest.mark.skipif( - [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2], - reason="uses meanings semantics from datastore v2.20.1 and lower", - ) @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_compressed_legacy(self): + def test__from_datastore_compressed_repeated_to_compressed(self): class ThisKind(model.Model): foo = model.BlobProperty(compressed=True, repeated=True) @@ -1972,7 +1968,7 @@ class ThisKind(model.Model): ], ) @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_compressed(self, meaning): + def test__from_datastore_compressed_repeated_to_compressed_tuple_meaning(self, meaning): class ThisKind(model.Model): foo = model.BlobProperty(compressed=True, repeated=True) @@ -1996,12 +1992,8 @@ class ThisKind(model.Model): ds_entity = model._entity_to_ds_entity(entity) assert ds_entity["foo"] == [compressed_value_one, compressed_value_two] - @pytest.mark.skipif( - [int(v) for v in datastore.__version__.split(".")] >= [2, 20, 2], - reason="uses meanings semantics from datastore v2.20.1 and lower", - ) @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_uncompressed_legacy(self): + def test__from_datastore_compressed_repeated_to_uncompressed(self): class ThisKind(model.Model): foo = model.BlobProperty(compressed=False, repeated=True) @@ -2039,7 +2031,7 @@ class ThisKind(model.Model): ], ) @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_uncompressed(self, meaning): + def test__from_datastore_compressed_repeated_to_uncompressed_tuple_meaning(self, meaning): class ThisKind(model.Model): foo = model.BlobProperty(compressed=False, repeated=True) From 5256febdf45855557871e382e4cf5c432c6b9bf4 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 21 Mar 2025 21:53:34 +0000 Subject: [PATCH 10/13] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/unit/test_model.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 0e17d6fb..42cd3358 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -1968,7 +1968,9 @@ class ThisKind(model.Model): ], ) @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_compressed_tuple_meaning(self, meaning): + def test__from_datastore_compressed_repeated_to_compressed_tuple_meaning( + self, meaning + ): class ThisKind(model.Model): foo = model.BlobProperty(compressed=True, repeated=True) @@ -2031,7 +2033,9 @@ class ThisKind(model.Model): ], ) @pytest.mark.usefixtures("in_context") - def test__from_datastore_compressed_repeated_to_uncompressed_tuple_meaning(self, meaning): + def test__from_datastore_compressed_repeated_to_uncompressed_tuple_meaning( + self, meaning + ): class ThisKind(model.Model): foo = model.BlobProperty(compressed=False, repeated=True) From 099b18e5d283af4a87c733b3b4e3692540b7906f Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 25 Apr 2025 15:21:59 -0700 Subject: [PATCH 11/13] added test for mismatched sizes --- tests/unit/test_model.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 42cd3358..258c5c8b 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -2148,6 +2148,39 @@ class ThisKind(model.Model): ds_entity = model._entity_to_ds_entity(entity) assert ds_entity["foo"] == [compressed_value_one, compressed_value_two] + @staticmethod + @pytest.mark.usefixtures("in_context") + def test__from_datastore_large_value_list(): + """ + try calling _from_datastore with a meaning list smaller than the value list + """ + + prop = model.BlobProperty(compressed=False, repeated=True) + prop._name = "foo" + + key = datastore.Key("ThisKind", 123, project="testing") + datastore_entity = datastore.Entity(key=key) + uncompressed_value_one = b"abc" * 1000 + compressed_value_one = zlib.compress(uncompressed_value_one) + uncompressed_value_two = b"xyz" * 1000 + compressed_value_two = zlib.compress(uncompressed_value_two) + compressed_value = [model._BaseValue(compressed_value_one), model._BaseValue(compressed_value_two)] + datastore_entity.update({"foo": compressed_value}) + meanings = { + "foo": ( + (None, [model._MEANING_COMPRESSED]), + compressed_value, + ) + } + + datastore_entity._meanings = meanings + + updated_value = prop._from_datastore(datastore_entity, compressed_value) + assert len(updated_value) == 2 + assert updated_value[0].b_val == uncompressed_value_one + # second value should remain compressed + assert updated_value[1].b_val == compressed_value_two + @staticmethod @pytest.mark.usefixtures("in_context") def test__from_datastore_uncompressed_to_uncompressed(): From e808874547388f47a593859a335f3d74e470e4b5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 25 Apr 2025 15:24:06 -0700 Subject: [PATCH 12/13] added test for mismatched meaning list size --- tests/unit/test_model.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index 258c5c8b..b642aa3b 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -2155,8 +2155,7 @@ def test__from_datastore_large_value_list(): try calling _from_datastore with a meaning list smaller than the value list """ - prop = model.BlobProperty(compressed=False, repeated=True) - prop._name = "foo" + prop = model.BlobProperty(compressed=False, repeated=True, name="foo") key = datastore.Key("ThisKind", 123, project="testing") datastore_entity = datastore.Entity(key=key) @@ -2164,7 +2163,10 @@ def test__from_datastore_large_value_list(): compressed_value_one = zlib.compress(uncompressed_value_one) uncompressed_value_two = b"xyz" * 1000 compressed_value_two = zlib.compress(uncompressed_value_two) - compressed_value = [model._BaseValue(compressed_value_one), model._BaseValue(compressed_value_two)] + compressed_value = [ + model._BaseValue(compressed_value_one), + model._BaseValue(compressed_value_two), + ] datastore_entity.update({"foo": compressed_value}) meanings = { "foo": ( From 6bca078bf34328aa3ec4a0b8dc4fd485104d190c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 25 Apr 2025 16:04:21 -0700 Subject: [PATCH 13/13] fixed docstring --- google/cloud/ndb/key.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/ndb/key.py b/google/cloud/ndb/key.py index 3c3af888..b168e55a 100644 --- a/google/cloud/ndb/key.py +++ b/google/cloud/ndb/key.py @@ -716,13 +716,13 @@ def reference(self): >>> key = ndb.Key("Trampoline", 88, project="xy", database="wv", namespace="zt") >>> key.reference() app: "xy" - name_space: "zt" path { element { type: "Trampoline" id: 88 } } + name_space: "zt" database_id: "wv" """