Skip to content

Commit 84bb4d7

Browse files
Yalin Liannatisch
andauthored
[Tables] Add TableEntity encoder (#28172)
* WIP * Merge encode keys test into basic key type test * Merge basic and complex keys type into one * Change type conversion test to live * Convert tuple and raw entity tests to live * Avoid edm type for int32, bool, string * Convert typical values test to live * Add identical encoder tests to cosmos * Update comments in encoder tests * Update encoder tests recordings * Merged encoder tests * Convert unit tests to live tests * Introduce generic type in encoder * Update sync APIs for the generic entity type support * Expose default encoder type * Address expected_entity in test_encoder.py * Update tests and adjust encoder to fit more scenarios * Fix issues when getting value from *args * Update backward compatibility check in tests * Remove prepare_key() call in default encode_entity() * Make default encoder tests for "create" pass * Update expected_entity for non-string key example * Add expected_backcompat_entity for "create" tests * Fix bug on encoding enum values * Make all test_encoder_create_XXX pass after adjustments * Remove custom encoder tests * Make all upsert tests pass * Make all update tests pass * Make delete and get tests pass * Make all cosmos sync tests pass * Update encoding on tuples * Update tests based on encoder updates * Update assets.json * Revert last updates in encoder for entity value in tuple type * Update assets.json * Update a cosmos test for intend changes from encoder * Update error value to compare * Not filtering out string "None" in new_encoding in _check_backcompat() * Return detected edm type for enum values * Add license header * pop values from encoded result * Update type hints * Add async changes * pop -> get * Update assets.json * Move tuple handle into prepare_value() * Update two failing storage edmtype tests * Run black * Able to check_backcompat for non-string key case * Fix mypy * Address types * Make tuple tests backward compatible * Update tests and recordings * Changed to get pk and rk from encoded entity in "delete" * Update tests and recordings * Fix pylint * Remove throw error for unknown edm type in "_prepare_value_in_tuple" * Fix IndexError in "delete" * Support encoder in "batch" * Add tests and recordings * Make batch encoder tests live only * Fix sphinx --------- Co-authored-by: antisch <[email protected]>
1 parent aa3a4b3 commit 84bb4d7

17 files changed

+8209
-2004
lines changed

sdk/tables/azure-data-tables/assets.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "python",
44
"TagPrefix": "python/tables/azure-data-tables",
5-
"Tag": "python/tables/azure-data-tables_4f55d47a10"
5+
"Tag": "python/tables/azure-data-tables_7ddb8a1cfc"
66
}

sdk/tables/azure-data-tables/azure/data/tables/__init__.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# Licensed under the MIT License. See License.txt in the project root for
44
# license information.
55
# --------------------------------------------------------------------------
6+
from ._encoder import TableEntityEncoder, TableEntityEncoderABC
67
from ._entity import TableEntity, EntityProperty, EdmType, EntityMetadata
78
from ._error import RequestTooLargeError, TableTransactionError, TableErrorCode
89
from ._table_shared_access_signature import generate_table_sas, generate_account_sas
@@ -31,6 +32,8 @@
3132
"TableServiceClient",
3233
"ResourceTypes",
3334
"AccountSasPermissions",
35+
"TableEntityEncoder",
36+
"TableEntityEncoderABC",
3437
"TableErrorCode",
3538
"TableSasPermissions",
3639
"TableAccessPolicy",
Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
# -------------------------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# Licensed under the MIT License. See License.txt in the project root for
4+
# license information.
5+
# --------------------------------------------------------------------------
6+
import abc
7+
import enum
8+
from typing import Any, Optional, Tuple, Mapping, Union, TypeVar, Generic, Dict
9+
from uuid import UUID
10+
from datetime import datetime
11+
from math import isnan
12+
13+
from ._entity import EdmType, TableEntity
14+
from ._deserialize import _convert_to_entity
15+
from ._common_conversion import _encode_base64, _to_utc_datetime
16+
17+
_ODATA_SUFFIX = "@odata.type"
18+
T = TypeVar("T")
19+
20+
21+
class TableEntityEncoderABC(abc.ABC, Generic[T]):
22+
def prepare_key(self, key: str) -> str:
23+
"""Duplicate the single quote char to escape.
24+
25+
:param str key: The entity PartitionKey or RowKey value in table entity.
26+
:return: The entity PartitionKey or RowKey value in table entity.
27+
:rtype: str
28+
"""
29+
try:
30+
return key.replace("'", "''")
31+
except (AttributeError, TypeError) as exc:
32+
raise TypeError("PartitionKey or RowKey must be of type string.") from exc
33+
34+
def prepare_value( # pylint: disable=too-many-return-statements
35+
self, name: Optional[str], value: Any
36+
) -> Tuple[Optional[EdmType], Optional[Union[str, int, float, bool]]]:
37+
"""Prepare the encoded value and its edm type.
38+
39+
:param name: The entity property name.
40+
:type name: str or None
41+
:param value: The entity property value.
42+
:type value: Any
43+
:return: The value edm type and encoded value.
44+
:rtype: A tuple of ~azure.data.tables.EdmType or None, and str, int, float, bool or None
45+
"""
46+
if isinstance(value, bool):
47+
return None, value
48+
if isinstance(value, enum.Enum):
49+
return self.prepare_value(name, value.value)
50+
if isinstance(value, str):
51+
return None, value
52+
if isinstance(value, int):
53+
return None, value # TODO: Test what happens if the supplied value exceeds int32.
54+
if isinstance(value, float):
55+
if isnan(value):
56+
return EdmType.DOUBLE, "NaN"
57+
if value == float("inf"):
58+
return EdmType.DOUBLE, "Infinity"
59+
if value == float("-inf"):
60+
return EdmType.DOUBLE, "-Infinity"
61+
return EdmType.DOUBLE, value
62+
if isinstance(value, UUID):
63+
return EdmType.GUID, str(value)
64+
if isinstance(value, bytes):
65+
return EdmType.BINARY, _encode_base64(value)
66+
if isinstance(value, datetime):
67+
try:
68+
if hasattr(value, "tables_service_value") and value.tables_service_value:
69+
return EdmType.DATETIME, value.tables_service_value
70+
except AttributeError:
71+
pass
72+
return EdmType.DATETIME, _to_utc_datetime(value)
73+
if isinstance(value, tuple):
74+
return self._prepare_value_in_tuple(value)
75+
if value is None:
76+
return None, None
77+
if name:
78+
raise TypeError(f"Unsupported data type '{type(value)}' for entity property '{name}'.")
79+
raise TypeError(f"Unsupported data type '{type(value)}'.")
80+
81+
def _prepare_value_in_tuple( # pylint: disable=too-many-return-statements
82+
self, value: Tuple[Any, Optional[Union[str, EdmType]]]
83+
) -> Tuple[Optional[EdmType], Optional[Union[str, int, float, bool]]]:
84+
unencoded_value = value[0]
85+
edm_type = value[1]
86+
if unencoded_value is None:
87+
return EdmType(edm_type), unencoded_value
88+
if edm_type == EdmType.STRING:
89+
return EdmType.STRING, str(unencoded_value)
90+
if edm_type == EdmType.INT64:
91+
return EdmType.INT64, str(unencoded_value)
92+
if edm_type == EdmType.INT32:
93+
return EdmType.INT32, int(unencoded_value)
94+
if edm_type == EdmType.BOOLEAN:
95+
return EdmType.BOOLEAN, unencoded_value
96+
if edm_type == EdmType.GUID:
97+
return EdmType.GUID, str(unencoded_value)
98+
if edm_type == EdmType.BINARY:
99+
# Leaving this with the double-encoding bug for now, as per original implementation
100+
return EdmType.BINARY, _encode_base64(unencoded_value)
101+
if edm_type == EdmType.DOUBLE:
102+
if isinstance(unencoded_value, str):
103+
# Pass a serialized value straight through
104+
return EdmType.DOUBLE, unencoded_value
105+
if isnan(unencoded_value):
106+
return EdmType.DOUBLE, "NaN"
107+
if unencoded_value == float("inf"):
108+
return EdmType.DOUBLE, "Infinity"
109+
if unencoded_value == float("-inf"):
110+
return EdmType.DOUBLE, "-Infinity"
111+
return EdmType.DOUBLE, unencoded_value
112+
if edm_type == EdmType.DATETIME:
113+
if isinstance(unencoded_value, str):
114+
# Pass a serialized datetime straight through
115+
return EdmType.DATETIME, unencoded_value
116+
try:
117+
# Check is this is a 'round-trip' datetime, and if so
118+
# pass through the original value.
119+
if unencoded_value.tables_service_value:
120+
return EdmType.DATETIME, unencoded_value.tables_service_value
121+
except AttributeError:
122+
pass
123+
return EdmType.DATETIME, _to_utc_datetime(unencoded_value)
124+
raise TypeError(f"Unsupported data type '{type(value)}'.")
125+
126+
@abc.abstractmethod
127+
def encode_entity(self, entity: T) -> Dict[str, Union[str, int, float, bool]]:
128+
"""Encode an entity object into JSON format to send out.
129+
130+
:param entity: A table entity.
131+
:type entity: Custom entity type
132+
:return: An entity with property's metadata in JSON format.
133+
:rtype: dict
134+
"""
135+
136+
@abc.abstractmethod
137+
def decode_entity(self, entity: Dict[str, Union[str, int, float, bool]]) -> T: ...
138+
139+
140+
class TableEntityEncoder(TableEntityEncoderABC[Union[TableEntity, Mapping[str, Any]]]):
141+
def encode_entity(self, entity: Union[TableEntity, Mapping[str, Any]]) -> Dict[str, Union[str, int, float, bool]]:
142+
"""Encode an entity object into JSON format to send out.
143+
The entity format is:
144+
145+
.. code-block:: json
146+
147+
{
148+
"Address":"Mountain View",
149+
"Age":23,
150+
"AmountDue":200.23,
151+
"[email protected]":"Edm.Guid",
152+
"CustomerCode":"c9da6455-213d-42c9-9a79-3e9149a57833",
153+
"[email protected]":"Edm.DateTime",
154+
"CustomerSince":"2008-07-10T00:00:00",
155+
"IsActive":true,
156+
"[email protected]":"Edm.Int64",
157+
"NumberOfOrders":"255",
158+
"PartitionKey":"my_partition_key",
159+
"RowKey":"my_row_key"
160+
}
161+
162+
:param entity: A table entity.
163+
:type entity: ~azure.data.tables.TableEntity or Mapping[str, Any]
164+
:return: An entity with property's metadata in JSON format.
165+
:rtype: dict
166+
"""
167+
encoded = {}
168+
for key, value in entity.items():
169+
edm_type, value = self.prepare_value(key, value)
170+
try:
171+
if _ODATA_SUFFIX in key or key + _ODATA_SUFFIX in entity:
172+
encoded[key] = value
173+
continue
174+
# The edm type is decided by value
175+
# For example, when value=EntityProperty(str(uuid.uuid4), "Edm.Guid"),
176+
# the type is string instead of Guid after encoded
177+
if edm_type:
178+
encoded[key + _ODATA_SUFFIX] = edm_type.value if hasattr(edm_type, "value") else edm_type
179+
except TypeError:
180+
pass
181+
encoded[key] = value
182+
return encoded
183+
184+
def decode_entity(self, entity: Dict[str, Union[str, int, float, bool]]) -> TableEntity:
185+
return _convert_to_entity(entity)

sdk/tables/azure-data-tables/azure/data/tables/_error.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919
from azure.core.pipeline.policies import ContentDecodePolicy
2020

2121

22-
_ERROR_TYPE_NOT_SUPPORTED = "Type not supported when sending data to the service: {0}."
23-
_ERROR_VALUE_TOO_LARGE = "{0} is too large to be cast to type {1}."
2422
_ERROR_UNKNOWN = "Unknown error ({0})"
2523
_ERROR_VALUE_NONE = "{0} should not be None."
2624

0 commit comments

Comments
 (0)