Skip to content

Commit 4f4bacf

Browse files
FabianMeiswinkelFabianMeiswinkel
authored
Adding system property to change serialization inclusion for item serialization only (Azure#44035)
* Addressing NPE for readItem returning 404 * Update CHANGELOG.md * Update SparkE2EGatewayChangeFeedITest.scala * Adding system property to influence serialization inclusion mode for only items * Added system property to allow overriding serialization inclusion mode for items/documents * Merging main * Update HttpConstants.java * Update GoneAndRetryWithRetryPolicy.java
1 parent 080ecba commit 4f4bacf

File tree

112 files changed

+858
-599
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

112 files changed

+858
-599
lines changed

‎sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/ReadMyWriteWorkflow.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55

66
import com.azure.cosmos.CosmosBridgeInternal;
77
import com.azure.cosmos.CosmosException;
8-
import com.azure.cosmos.CosmosItemSerializer;
98
import com.azure.cosmos.implementation.AsyncDocumentClient;
109
import com.azure.cosmos.implementation.CosmosPagedFluxOptions;
1110
import com.azure.cosmos.implementation.Database;
@@ -194,12 +193,12 @@ private Flux<Document> writeDocument(Integer i) {
194193
String randomVal = UUID.randomUUID().toString();
195194
Document document = new Document();
196195
document.setId(idString);
197-
document.set(partitionKey, idString, CosmosItemSerializer.DEFAULT_SERIALIZER);
198-
document.set(QUERY_FIELD_NAME, randomVal, CosmosItemSerializer.DEFAULT_SERIALIZER);
199-
document.set("dataField1", randomVal, CosmosItemSerializer.DEFAULT_SERIALIZER);
200-
document.set("dataField2", randomVal, CosmosItemSerializer.DEFAULT_SERIALIZER);
201-
document.set("dataField3", randomVal, CosmosItemSerializer.DEFAULT_SERIALIZER);
202-
document.set("dataField4", randomVal, CosmosItemSerializer.DEFAULT_SERIALIZER);
196+
document.set(partitionKey, idString);
197+
document.set(QUERY_FIELD_NAME, randomVal);
198+
document.set("dataField1", randomVal);
199+
document.set("dataField2", randomVal);
200+
document.set("dataField3", randomVal);
201+
document.set("dataField4", randomVal);
203202

204203
Integer key = i == null ? cacheKey() : i;
205204
return client.createDocument(getCollectionLink(), document, null, false)

‎sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/models/SqlQuerySpecWithEncryption.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import com.azure.cosmos.encryption.implementation.EncryptionUtils;
1212
import com.azure.cosmos.encryption.implementation.mdesrc.cryptography.EncryptionType;
1313
import com.azure.cosmos.encryption.implementation.mdesrc.cryptography.MicrosoftDataEncryptionException;
14+
import com.azure.cosmos.implementation.DefaultCosmosItemSerializer;
1415
import com.azure.cosmos.implementation.Utils;
1516
import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
1617
import com.azure.cosmos.implementation.apachecommons.lang.tuple.Pair;
@@ -84,9 +85,11 @@ Mono<Void> addEncryptionParameterAsync(String path, SqlParameter sqlParameter,
8485
}
8586
}
8687
byte[] valueByte =
87-
EncryptionUtils.serializeJsonToByteArray(CosmosItemSerializer.DEFAULT_SERIALIZER,
88+
EncryptionUtils.serializeJsonToByteArray(
89+
DefaultCosmosItemSerializer.INTERNAL_DEFAULT_SERIALIZER,
8890
sqlParameter.getValue(Object.class));
89-
JsonNode itemJObj = Utils.parse(valueByte, JsonNode.class, CosmosItemSerializer.DEFAULT_SERIALIZER);
91+
JsonNode itemJObj = Utils.parse(
92+
valueByte, JsonNode.class, DefaultCosmosItemSerializer.INTERNAL_DEFAULT_SERIALIZER);
9093
Pair<EncryptionProcessor.TypeMarker, byte[]> typeMarkerPair =
9194
EncryptionProcessor.toByteArray(itemJObj);
9295
byte[] cipherText =

‎sdk/cosmos/azure-cosmos-spark_3_2-12/src/main/scala/com/azure/cosmos/CosmosItemSerializerNoExceptionWrapping.scala‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,11 @@
22
// Licensed under the MIT License.
33
package com.azure.cosmos
44

5-
private[cosmos] abstract class CosmosItemSerializerNoExceptionWrapping extends CosmosItemSerializer(false) {
5+
import com.azure.cosmos.implementation.ImplementationBridgeHelpers
6+
7+
private[cosmos] abstract class CosmosItemSerializerNoExceptionWrapping extends CosmosItemSerializer() {
8+
ImplementationBridgeHelpers
9+
.CosmosItemSerializerHelper
10+
.getCosmosItemSerializerAccessor
11+
.setShouldWrapSerializationExceptions(this, false)
612
}

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ClientMetricsTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -604,7 +604,7 @@ public void replaceItem() throws Exception {
604604

605605
validateItemResponse(properties, itemResponse);
606606
String newPropValue = UUID.randomUUID().toString();
607-
properties.set("newProp", newPropValue, CosmosItemSerializer.DEFAULT_SERIALIZER);
607+
properties.set("newProp", newPropValue);
608608
CosmosItemRequestOptions options = new CosmosItemRequestOptions();
609609
ModelBridgeInternal.setPartitionKey(options, new PartitionKey(properties.get("mypk")));
610610
// replace document

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ClientTelemetryTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@ private InternalObjectNode getInternalObjectNode() {
430430
InternalObjectNode internalObjectNode = new InternalObjectNode();
431431
String uuid = UUID.randomUUID().toString();
432432
internalObjectNode.setId(uuid);
433-
internalObjectNode.set("mypk", uuid, CosmosItemSerializer.DEFAULT_SERIALIZER);
433+
internalObjectNode.set("mypk", uuid);
434434
return internalObjectNode;
435435
}
436436

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1833,15 +1833,15 @@ private InternalObjectNode getInternalObjectNode() {
18331833
InternalObjectNode internalObjectNode = new InternalObjectNode();
18341834
String uuid = UUID.randomUUID().toString();
18351835
internalObjectNode.setId(uuid);
1836-
internalObjectNode.set("mypk", uuid, CosmosItemSerializer.DEFAULT_SERIALIZER);
1836+
internalObjectNode.set("mypk", uuid);
18371837
return internalObjectNode;
18381838
}
18391839

18401840
private InternalObjectNode getInternalObjectNode(String pkValue) {
18411841
InternalObjectNode internalObjectNode = new InternalObjectNode();
18421842
String uuid = UUID.randomUUID().toString();
18431843
internalObjectNode.setId(uuid);
1844-
internalObjectNode.set( "mypk", pkValue, CosmosItemSerializer.DEFAULT_SERIALIZER);
1844+
internalObjectNode.set( "mypk", pkValue);
18451845
return internalObjectNode;
18461846
}
18471847

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosItemContentResponseOnWriteTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ public void replaceItem_withContentResponseOnWriteDisabled() {
126126

127127
validateMinimalItemResponse(properties, itemResponse, true);
128128
String newPropValue = UUID.randomUUID().toString();
129-
properties.set("newProp", newPropValue, CosmosItemSerializer.DEFAULT_SERIALIZER);
129+
properties.set("newProp", newPropValue);
130130
ModelBridgeInternal.setPartitionKey(cosmosItemRequestOptions,
131131
new PartitionKey(properties.get("mypk")));
132132
// replace document
@@ -148,7 +148,7 @@ public void replaceItem_withContentResponseOnWriteEnabledThroughRequestOptions()
148148

149149
validateItemResponse(properties, itemResponse);
150150
String newPropValue = UUID.randomUUID().toString();
151-
properties.set("newProp", newPropValue, CosmosItemSerializer.DEFAULT_SERIALIZER);
151+
properties.set("newProp", newPropValue);
152152
ModelBridgeInternal.setPartitionKey(cosmosItemRequestOptions,
153153
new PartitionKey(properties.get("mypk")));
154154
// replace document

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosItemSerializerNoExceptionWrapping.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,13 @@
66

77
package com.azure.cosmos;
88

9+
import com.azure.cosmos.implementation.ImplementationBridgeHelpers;
10+
911
public abstract class CosmosItemSerializerNoExceptionWrapping extends CosmosItemSerializer {
1012
public CosmosItemSerializerNoExceptionWrapping() {
11-
super(false);
13+
ImplementationBridgeHelpers
14+
.CosmosItemSerializerHelper
15+
.getCosmosItemSerializerAccessor()
16+
.setShouldWrapSerializationExceptions(this, false);
1217
}
1318
}

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosItemSerializerUnitTests.java‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
package com.azure.cosmos;
2+
3+
import com.azure.cosmos.implementation.Configs;
4+
import com.azure.cosmos.implementation.DefaultCosmosItemSerializer;
5+
import com.azure.cosmos.implementation.Utils;
6+
import com.fasterxml.jackson.databind.ObjectMapper;
7+
import com.fasterxml.jackson.databind.node.ObjectNode;
8+
import org.slf4j.Logger;
9+
import org.slf4j.LoggerFactory;
10+
import org.testng.annotations.DataProvider;
11+
import org.testng.annotations.Test;
12+
13+
import java.nio.ByteBuffer;
14+
import java.nio.charset.StandardCharsets;
15+
import java.util.ArrayList;
16+
import java.util.List;
17+
import java.util.Map;
18+
19+
import static org.assertj.core.api.Assertions.assertThat;
20+
21+
public class CosmosItemSerializerUnitTests {
22+
protected static Logger logger = LoggerFactory.getLogger(CosmosItemSerializerUnitTests.class.getSimpleName());
23+
@DataProvider
24+
public static Object[][] serializationInclusionModeDataProvider() {
25+
List<Object[]> providers = new ArrayList<>();
26+
27+
providers.add(
28+
new Object[] {
29+
"",
30+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0,\"nullValueProperty\":null,\"emptyValueProperty\":\"\"}"
31+
});
32+
providers.add(
33+
new Object[] {
34+
null,
35+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0,\"nullValueProperty\":null,\"emptyValueProperty\":\"\"}"
36+
}
37+
);
38+
providers.add(
39+
new Object[] {
40+
"Always",
41+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0,\"nullValueProperty\":null,\"emptyValueProperty\":\"\"}"
42+
}
43+
);
44+
providers.add(
45+
new Object[] {
46+
"NonNull",
47+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0,\"emptyValueProperty\":\"\"}"
48+
}
49+
);
50+
providers.add(
51+
new Object[] {
52+
"NOnNULl",
53+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0,\"emptyValueProperty\":\"\"}"
54+
}
55+
);
56+
providers.add(
57+
new Object[] {
58+
"NonDefault",
59+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0,\"emptyValueProperty\":\"\"}"
60+
}
61+
);
62+
providers.add(
63+
new Object[] {
64+
"NonEmpty",
65+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0}"
66+
}
67+
);
68+
69+
Object[][] array = new Object[providers.size()][];
70+
71+
return providers.toArray(array);
72+
}
73+
74+
@DataProvider
75+
public static Object[][] serializationInclusionModePojoDataProvider() {
76+
List<Object[]> providers = new ArrayList<>();
77+
78+
providers.add(
79+
new Object[] {
80+
"",
81+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0,\"nullValueProperty\":null,\"emptyValueProperty\":\"\"}"
82+
});
83+
providers.add(
84+
new Object[] {
85+
null,
86+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0,\"nullValueProperty\":null,\"emptyValueProperty\":\"\"}"
87+
}
88+
);
89+
providers.add(
90+
new Object[] {
91+
"Always",
92+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0,\"nullValueProperty\":null,\"emptyValueProperty\":\"\"}"
93+
}
94+
);
95+
providers.add(
96+
new Object[] {
97+
"NonNull",
98+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0,\"emptyValueProperty\":\"\"}"
99+
}
100+
);
101+
providers.add(
102+
new Object[] {
103+
"NOnNULl",
104+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0,\"emptyValueProperty\":\"\"}"
105+
}
106+
);
107+
providers.add(
108+
new Object[] {
109+
"NonDefault",
110+
"{\"id\":\"SomeId\"}"
111+
}
112+
);
113+
providers.add(
114+
new Object[] {
115+
"NonEmpty",
116+
"{\"id\":\"SomeId\",\"defaultValueProperty\":0}"
117+
}
118+
);
119+
120+
Object[][] array = new Object[providers.size()][];
121+
122+
return providers.toArray(array);
123+
}
124+
125+
@Test(groups = { "unit" }, dataProvider = "serializationInclusionModeDataProvider")
126+
public void serializationInclusionModesTests(
127+
String systemPropertyValue,
128+
String expectedOutcome
129+
) {
130+
try {
131+
if (systemPropertyValue != null) {
132+
System.setProperty(Configs.ITEM_SERIALIZATION_INCLUSION_MODE, systemPropertyValue);
133+
}
134+
runSerializationInclusionModeTest(expectedOutcome);
135+
} finally {
136+
System.clearProperty(Configs.ITEM_SERIALIZATION_INCLUSION_MODE);
137+
}
138+
}
139+
140+
@Test(groups = { "unit" }, dataProvider = "serializationInclusionModePojoDataProvider")
141+
public void serializationInclusionModesPojoTests(
142+
String systemPropertyValue,
143+
String expectedOutcome
144+
) {
145+
try {
146+
if (systemPropertyValue != null) {
147+
System.setProperty(Configs.ITEM_SERIALIZATION_INCLUSION_MODE, systemPropertyValue);
148+
}
149+
runSerializationInclusionModePojoTest(expectedOutcome);
150+
} finally {
151+
System.clearProperty(Configs.ITEM_SERIALIZATION_INCLUSION_MODE);
152+
}
153+
}
154+
155+
private void runSerializationInclusionModeTest(String expectedOutcome) {
156+
157+
ObjectMapper mapper = Utils.getDocumentObjectMapper(Configs.getItemSerializationInclusionMode());
158+
CosmosItemSerializer serializer = new DefaultCosmosItemSerializer(mapper);
159+
160+
ObjectNode node = Utils.getSimpleObjectMapper().createObjectNode();
161+
node.put("id", "SomeId");
162+
node.put("defaultValueProperty", 0);
163+
node.put("nullValueProperty", (String)null);
164+
node.put("emptyValueProperty", "");
165+
Map<String, Object> jsonMap = serializer.serialize(node);
166+
ByteBuffer buffer = Utils.serializeJsonToByteBuffer(
167+
serializer,
168+
jsonMap,
169+
null,
170+
false);
171+
String json = new String(buffer.array(), 0, buffer.limit(), StandardCharsets.UTF_8);
172+
logger.info("Actual: {}, Expected: {}", json, expectedOutcome);
173+
assertThat(json).isEqualTo(expectedOutcome);
174+
}
175+
176+
private void runSerializationInclusionModePojoTest(String expectedOutcome) {
177+
178+
ObjectMapper mapper = Utils.getDocumentObjectMapper(Configs.getItemSerializationInclusionMode());
179+
CosmosItemSerializer serializer = new DefaultCosmosItemSerializer(mapper);
180+
181+
PojoItem node = new PojoItem();
182+
Map<String, Object> jsonMap = serializer.serialize(node);
183+
ByteBuffer buffer = Utils.serializeJsonToByteBuffer(
184+
serializer,
185+
jsonMap,
186+
null,
187+
false);
188+
String json = new String(buffer.array(), 0, buffer.limit(), StandardCharsets.UTF_8);
189+
logger.info("Actual: {}, Expected: {}", json, expectedOutcome);
190+
assertThat(json).isEqualTo(expectedOutcome);
191+
}
192+
193+
private static class PojoItem
194+
{
195+
public String id = "SomeId";
196+
public int defaultValueProperty = 0;
197+
public String nullValueProperty = null;
198+
public String emptyValueProperty = "";
199+
}
200+
}

‎sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosItemTest.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@
5959
import java.util.Arrays;
6060
import java.util.HashMap;
6161
import java.util.HashSet;
62-
import java.util.Iterator;
6362
import java.util.List;
6463
import java.util.Map;
6564
import java.util.Set;
@@ -140,7 +139,7 @@ public void createLargeItem() throws Exception {
140139

141140
//Keep size as ~ 1.5MB to account for size of other props
142141
int size = (int) (ONE_MB * 1.5);
143-
docDefinition.set("largeString", StringUtils.repeat("x", size), CosmosItemSerializer.DEFAULT_SERIALIZER);
142+
docDefinition.set("largeString", StringUtils.repeat("x", size));
144143

145144
CosmosItemResponse<InternalObjectNode> itemResponse = container.createItem(docDefinition, new CosmosItemRequestOptions());
146145

@@ -154,7 +153,7 @@ public void createItemWithVeryLargePartitionKey() throws Exception {
154153
for(int i = 0; i < 100; i++) {
155154
sb.append(i).append("x");
156155
}
157-
docDefinition.set("mypk", sb.toString(), CosmosItemSerializer.DEFAULT_SERIALIZER);
156+
docDefinition.set("mypk", sb.toString());
158157

159158
CosmosItemResponse<InternalObjectNode> itemResponse = container.createItem(docDefinition, new CosmosItemRequestOptions());
160159

@@ -168,7 +167,7 @@ public void readItemWithVeryLargePartitionKey() throws Exception {
168167
for(int i = 0; i < 100; i++) {
169168
sb.append(i).append("x");
170169
}
171-
docDefinition.set("mypk", sb.toString(), CosmosItemSerializer.DEFAULT_SERIALIZER);
170+
docDefinition.set("mypk", sb.toString());
172171

173172
CosmosItemResponse<InternalObjectNode> itemResponse = container.createItem(docDefinition);
174173

@@ -1100,7 +1099,7 @@ public void replaceItem() throws Exception{
11001099

11011100
validateItemResponse(properties, itemResponse);
11021101
String newPropValue = UUID.randomUUID().toString();
1103-
properties.set("newProp", newPropValue, CosmosItemSerializer.DEFAULT_SERIALIZER);
1102+
properties.set("newProp", newPropValue);
11041103
CosmosItemRequestOptions options = new CosmosItemRequestOptions();
11051104
ModelBridgeInternal.setPartitionKey(options, new PartitionKey(properties.get("mypk")));
11061105
// replace document

0 commit comments

Comments
 (0)