Skip to content

Commit 3419944

Browse files
jeet1995jeet1995
authored
Fault injection with Gateway V2 (Azure#46223)
* Mapping URI to RegionalRoutingContext. * Adding tests. * Adding tests. * Adding tests. * Adding tests. * Adding tests. * Adding tests. * Adding tests. * Revert ConnectionType and add readMany tests. * Assert Gateway v2 usage. * Assert Gateway v2 + LatestVersion change feed integration. * Refactoring * Prepare for Gateway v2 multi-writer support in pipeline. * Prepare for Gateway v2 multi-writer support in pipeline. * Fixing tests. * Adding samples. * Addressing code comments. * Fixing CHANGELOG.md. * Addressing review comments. * Update code snippet. * Fixing tests. * Fixing tests. * Fixing tests. * Addressing review comments. * Logging changes * Adding live test config files for Gateway v2.0 * Adding live test config files for Gateway v2.0 * Fixing live tests pipeline. * Adding live test config files for Gateway v2.0 * Fixing compilation errors. * Addressing review comments. * Addressing review comments. * Addressing review comments.
1 parent 529297e commit 3419944

33 files changed

+1940
-108
lines changed

‎sdk/cosmos/azure-cosmos-test/CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
### 1.0.0-beta.14 (Unreleased)
44

55
#### Features Added
6+
* Added support for Gateway V2 - See [PR 46223](https://github.com/Azure/azure-sdk-for-java/pull/46223)
67

78
#### Breaking Changes
89

‎sdk/cosmos/azure-cosmos-test/README.md‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ The following section provides several code snippets covering how to create some
3636
* [Broken Connection Scenario](#broken-connection-scenario)
3737
* [Server Return Gone Scenario](#server-return-gone-scenario)
3838
* [Random Connection Close Scenario](#random-connection-close-scenario)
39+
* [Connection Delay Scenario (Gateway V2)](#connection-delay-scenario-gateway-v2)
40+
* [Response Delay Scenario (Gateway V2)](#response-delay-scenario-gateway-v2)
41+
* [Service Unavailable Scenario (Gateway V2)](#service-unavailable-scenario-gateway-v2)
3942

4043
### High Channel Acquisition Scenario
4144

@@ -126,6 +129,146 @@ FaultInjectionRule connectionErrorRule =
126129
CosmosFaultInjectionHelper.configureFaultInjectionRules(container, Arrays.asList(connectionErrorRule)).block();
127130
```
128131

132+
### Connection Delay Scenario (Gateway V2)
133+
134+
```java readme-sample-connectionDelayWithGatewayV2Scenario
135+
// Enable thin client and configure HTTP/2
136+
System.setProperty("COSMOS.THINCLIENT_ENABLED", "true");
137+
138+
CosmosAsyncClient gatewayV2AsyncClient = new CosmosClientBuilder()
139+
.endpoint("<YOUR ENDPOINT HERE>")
140+
.key("<YOUR KEY HERE>")
141+
.contentResponseOnWriteEnabled(true)
142+
.gatewayMode(new GatewayConnectionConfig().setHttp2ConnectionConfig(new Http2ConnectionConfig().setEnabled(true)))
143+
.buildAsyncClient();
144+
145+
CosmosAsyncContainer container = gatewayV2AsyncClient
146+
.getDatabase("<YOUR DATABASE NAME>")
147+
.getContainer("<YOUR CONTAINER NAME>");
148+
149+
// Define fault injection rule for connection delay
150+
FaultInjectionRule connectionDelayRule = new FaultInjectionRuleBuilder("connection-delay-rule")
151+
.condition(new FaultInjectionConditionBuilder()
152+
.operationType(FaultInjectionOperationType.READ_ITEM)
153+
.connectionType(FaultInjectionConnectionType.GATEWAY)
154+
.build())
155+
.result(FaultInjectionResultBuilders
156+
.getResultBuilder(FaultInjectionServerErrorType.CONNECTION_DELAY)
157+
.delay(Duration.ofSeconds(8))
158+
.times(1)
159+
.build())
160+
.duration(Duration.ofMinutes(5))
161+
.build();
162+
163+
try {
164+
// Apply fault injection rule
165+
CosmosFaultInjectionHelper.configureFaultInjectionRules(container, Arrays.asList(connectionDelayRule)).block();
166+
167+
// Trigger fault injection by performing a read operation
168+
container.readItem("<ITEM_ID>", new PartitionKey("<PARTITION_KEY>"), Object.class).block();
169+
} finally {
170+
// Clean up
171+
connectionDelayRule.disable();
172+
System.clearProperty("COSMOS.THINCLIENT_ENABLED");
173+
gatewayV2AsyncClient.close();
174+
}
175+
```
176+
177+
### Response Delay Scenario (Gateway V2)
178+
179+
```java readme-sample-responseDelayWithGatewayV2Scenario
180+
// Enable thin client and configure HTTP/2
181+
System.setProperty("COSMOS.THINCLIENT_ENABLED", "true");
182+
183+
CosmosAsyncClient gatewayV2AsyncClient = new CosmosClientBuilder()
184+
.endpoint("<YOUR ENDPOINT HERE>")
185+
.key("<YOUR KEY HERE>")
186+
.contentResponseOnWriteEnabled(true)
187+
.gatewayMode(new GatewayConnectionConfig().setHttp2ConnectionConfig(new Http2ConnectionConfig().setEnabled(true)))
188+
.buildAsyncClient();
189+
190+
CosmosAsyncContainer container = gatewayV2AsyncClient
191+
.getDatabase("<YOUR DATABASE NAME>")
192+
.getContainer("<YOUR CONTAINER NAME>");
193+
194+
// Define fault injection rule for response delay
195+
FaultInjectionRule responseDelayRule = new FaultInjectionRuleBuilder("response-delay-rule")
196+
.condition(new FaultInjectionConditionBuilder()
197+
.operationType(FaultInjectionOperationType.READ_ITEM)
198+
.connectionType(FaultInjectionConnectionType.GATEWAY)
199+
.build())
200+
.result(FaultInjectionResultBuilders
201+
.getResultBuilder(FaultInjectionServerErrorType.RESPONSE_DELAY)
202+
.delay(Duration.ofSeconds(10))
203+
.times(1)
204+
.build())
205+
.duration(Duration.ofMinutes(5))
206+
.build();
207+
208+
try {
209+
// Apply fault injection rule
210+
CosmosFaultInjectionHelper.configureFaultInjectionRules(container, Arrays.asList(responseDelayRule)).block();
211+
212+
// Trigger fault injection by performing a read operation
213+
container.readItem("<ITEM_ID>", new PartitionKey("<PARTITION_KEY>"), Object.class).block();
214+
} finally {
215+
// Clean up
216+
responseDelayRule.disable();
217+
System.clearProperty("COSMOS.THINCLIENT_ENABLED");
218+
gatewayV2AsyncClient.close();
219+
}
220+
```
221+
222+
### Service Unavailable Scenario (Gateway V2)
223+
224+
```java readme-sample-serviceUnavailableWithGatewayV2Scenario
225+
// Enable thin client and configure HTTP/2
226+
System.setProperty("COSMOS.THINCLIENT_ENABLED", "true");
227+
228+
CosmosAsyncClient gatewayV2AsyncClient = new CosmosClientBuilder()
229+
.endpoint("<YOUR ENDPOINT HERE>")
230+
.key("<YOUR KEY HERE>")
231+
.gatewayMode(new GatewayConnectionConfig().setHttp2ConnectionConfig(new Http2ConnectionConfig().setEnabled(true)))
232+
.contentResponseOnWriteEnabled(true)
233+
.buildAsyncClient();
234+
235+
CosmosAsyncContainer container = gatewayV2AsyncClient
236+
.getDatabase("<YOUR DATABASE NAME>")
237+
.getContainer("<YOUR CONTAINER NAME>");
238+
239+
// Define fault injection rule for service unavailable
240+
FaultInjectionRule serviceUnavailableRule = new FaultInjectionRuleBuilder("service-unavailable-rule")
241+
.condition(new FaultInjectionConditionBuilder()
242+
.operationType(FaultInjectionOperationType.READ_ITEM)
243+
.connectionType(FaultInjectionConnectionType.GATEWAY)
244+
.build())
245+
.result(FaultInjectionResultBuilders
246+
.getResultBuilder(FaultInjectionServerErrorType.SERVICE_UNAVAILABLE)
247+
.times(1)
248+
.build())
249+
.duration(Duration.ofMinutes(5))
250+
.build();
251+
252+
try {
253+
// Apply fault injection rule
254+
CosmosFaultInjectionHelper.configureFaultInjectionRules(container, Arrays.asList(serviceUnavailableRule)).block();
255+
256+
try {
257+
// Trigger fault injection by performing a read operation
258+
container.readItem("<ITEM_ID>", new PartitionKey("<PARTITION_KEY>"), Object.class).block();
259+
} catch (CosmosException e) {
260+
// Log diagnostics if fault injection causes failure
261+
CosmosDiagnostics diagnostics = e.getDiagnostics();
262+
System.out.println("Fault injection triggered: " + diagnostics);
263+
}
264+
} finally {
265+
// Clean up
266+
serviceUnavailableRule.disable();
267+
System.clearProperty("COSMOS.THINCLIENT_ENABLED");
268+
gatewayV2AsyncClient.close();
269+
}
270+
```
271+
129272
## Troubleshooting
130273

131274
### General

‎sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/faultinjection/FaultInjectionRule.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ public List<String> getAddresses() {
158158
public List<String> getRegionEndpoints() {
159159
return this.effectiveRule == null
160160
? null
161-
: this.effectiveRule.getRegionEndpoints().stream().map(URI::toString).collect(Collectors.toList());
161+
: this.effectiveRule.getRegionalRoutingContexts().stream().map(regionalRoutingContext -> regionalRoutingContext.getGatewayRegionalEndpoint().toString()).collect(Collectors.toList());
162162
}
163163

164164
/***

‎sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionConditionInternal.java‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import com.azure.cosmos.implementation.Utils;
99
import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
1010
import com.azure.cosmos.implementation.faultinjection.FaultInjectionRequestArgs;
11+
import com.azure.cosmos.implementation.routing.RegionalRoutingContext;
1112

1213
import java.net.URI;
1314
import java.util.ArrayList;
@@ -18,7 +19,7 @@ public class FaultInjectionConditionInternal {
1819
private final String containerResourceId;
1920
private final String containerName;
2021
private OperationType operationType;
21-
private List<URI> regionEndpoints;
22+
private List<RegionalRoutingContext> regionalRoutingContexts;
2223
private List<URI> physicalAddresses;
2324
private List<IFaultInjectionConditionValidator> validators;
2425

@@ -46,15 +47,15 @@ public void setResourceType(ResourceType resourceType) {
4647
}
4748
}
4849

49-
public void setRegionEndpoints(List<URI> regionEndpoints) {
50-
this.regionEndpoints = regionEndpoints;
51-
if (this.regionEndpoints != null) {
52-
this.validators.add(new RegionEndpointValidator(this.regionEndpoints));
50+
public void setRegionalRoutingContexts(List<RegionalRoutingContext> regionalRoutingContexts) {
51+
this.regionalRoutingContexts = regionalRoutingContexts;
52+
if (this.regionalRoutingContexts != null) {
53+
this.validators.add(new RegionEndpointValidator(this.regionalRoutingContexts));
5354
}
5455
}
5556

56-
public List<URI> getRegionEndpoints() {
57-
return this.regionEndpoints;
57+
public List<RegionalRoutingContext> getRegionalRoutingContexts() {
58+
return this.regionalRoutingContexts;
5859
}
5960

6061
public List<URI> getAddresses() {
@@ -94,22 +95,22 @@ interface IFaultInjectionConditionValidator {
9495
}
9596

9697
static class RegionEndpointValidator implements IFaultInjectionConditionValidator {
97-
private List<URI> regionEndpoints;
98-
RegionEndpointValidator(List<URI> regionEndpoints) {
99-
this.regionEndpoints = regionEndpoints;
98+
private List<RegionalRoutingContext> regionalRoutingContexts;
99+
RegionEndpointValidator(List<RegionalRoutingContext> regionalRoutingContexts) {
100+
this.regionalRoutingContexts = regionalRoutingContexts;
100101
}
101102
@Override
102103
public boolean isApplicable(String ruleId, FaultInjectionRequestArgs requestArgs) {
103104
boolean isApplicable =
104-
this.regionEndpoints.contains(requestArgs.getServiceRequest().faultInjectionRequestContext.getLocationEndpointToRoute());
105+
this.regionalRoutingContexts.contains(requestArgs.getServiceRequest().faultInjectionRequestContext.getRegionalRoutingContextToRoute());
105106
if (!isApplicable) {
106107
requestArgs.getServiceRequest().faultInjectionRequestContext
107108
.recordFaultInjectionRuleEvaluation(requestArgs.getTransportRequestId(),
108109
String.format(
109-
"%s [RegionEndpoint mismatch: Expected [%s], Actual [%s]]",
110+
"%s [RegionalRoutingContext mismatch: Expected [%s], Actual [%s]]",
110111
ruleId,
111-
this.regionEndpoints.stream().map(URI::toString).collect(Collectors.toList()),
112-
requestArgs.getServiceRequest().faultInjectionRequestContext.getLocationEndpointToRoute()));
112+
this.regionalRoutingContexts.stream().map(RegionalRoutingContext::toString).collect(Collectors.toList()),
113+
requestArgs.getServiceRequest().faultInjectionRequestContext.getRegionalRoutingContextToRoute()));
113114
}
114115

115116
return isApplicable;

‎sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionConnectionErrorRule.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package com.azure.cosmos.test.implementation.faultinjection;
55

66
import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
7+
import com.azure.cosmos.implementation.routing.RegionalRoutingContext;
78
import com.azure.cosmos.test.faultinjection.FaultInjectionConnectionErrorResult;
89
import com.azure.cosmos.test.faultinjection.FaultInjectionConnectionType;
910

@@ -22,7 +23,7 @@ public class FaultInjectionConnectionErrorRule implements IFaultInjectionRuleInt
2223
private final Instant startTime;
2324
private final Instant expireTime;
2425
private final AtomicLong hitCount;
25-
private final List<URI> regionEndpoints;
26+
private final List<RegionalRoutingContext> regionalRoutingContexts;
2627
private final List<URI> addresses;
2728
private final FaultInjectionConnectionType connectionType;
2829
private final FaultInjectionConnectionErrorResult result;
@@ -34,7 +35,7 @@ public FaultInjectionConnectionErrorRule(
3435
boolean enabled,
3536
Duration delay,
3637
Duration duration,
37-
List<URI> regionEndpoints,
38+
List<RegionalRoutingContext> regionalRoutingContexts,
3839
List<URI> addresses,
3940
FaultInjectionConnectionType connectionType,
4041
FaultInjectionConnectionErrorResult result) {
@@ -47,7 +48,7 @@ public FaultInjectionConnectionErrorRule(
4748
this.enabled = enabled;
4849
this.startTime = delay == null ? Instant.now() : Instant.now().plusMillis(delay.toMillis());
4950
this.expireTime = duration == null ? Instant.MAX : this.startTime.plusMillis(duration.toMillis());
50-
this.regionEndpoints = regionEndpoints;
51+
this.regionalRoutingContexts = regionalRoutingContexts;
5152
this.addresses = addresses;
5253
this.result = result;
5354
this.hitCount = new AtomicLong(0);
@@ -89,8 +90,8 @@ public List<URI> getAddresses() {
8990
}
9091

9192
@Override
92-
public List<URI> getRegionEndpoints() {
93-
return this.regionEndpoints;
93+
public List<RegionalRoutingContext> getRegionalRoutingContexts() {
94+
return this.regionalRoutingContexts;
9495
}
9596

9697
@Override

‎sdk/cosmos/azure-cosmos-test/src/main/java/com/azure/cosmos/test/implementation/faultinjection/FaultInjectionRuleProcessor.java‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -166,14 +166,14 @@ private Mono<IFaultInjectionRuleInternal> getEffectiveServerErrorRule(
166166
effectiveCondition.setResourceType(this.getEffectiveResourceType(rule.getCondition().getOperationType()));
167167
}
168168

169-
List<URI> regionEndpoints = this.getRegionEndpoints(rule.getCondition());
169+
List<RegionalRoutingContext> regionalRoutingContexts = this.getRegionalRoutingContexts(rule.getCondition());
170170
if (StringUtils.isEmpty(rule.getCondition().getRegion())) {
171171
// if region is not specific configured, then also add the defaultEndpoint
172-
List<URI> regionEndpointsWithDefault = new ArrayList<>(regionEndpoints);
173-
regionEndpointsWithDefault.add(this.globalEndpointManager.getDefaultEndpoint());
174-
effectiveCondition.setRegionEndpoints(regionEndpointsWithDefault);
172+
List<RegionalRoutingContext> regionalRoutingContextsWithDefault = new ArrayList<>(regionalRoutingContexts);
173+
regionalRoutingContextsWithDefault.add(new RegionalRoutingContext(this.globalEndpointManager.getDefaultEndpoint()));
174+
effectiveCondition.setRegionalRoutingContexts(regionalRoutingContextsWithDefault);
175175
} else {
176-
effectiveCondition.setRegionEndpoints(regionEndpoints);
176+
effectiveCondition.setRegionalRoutingContexts(regionalRoutingContexts);
177177
}
178178

179179
if (rule.getCondition().getConnectionType() == FaultInjectionConnectionType.GATEWAY) {
@@ -198,7 +198,7 @@ private Mono<IFaultInjectionRuleInternal> getEffectiveServerErrorRule(
198198
boolean primaryAddressesOnly = this.isWriteOnly(rule.getCondition());
199199
return BackoffRetryUtility.executeRetry(
200200
() -> this.resolvePhysicalAddresses(
201-
regionEndpoints,
201+
regionalRoutingContexts,
202202
rule.getCondition().getEndpoints(),
203203
primaryAddressesOnly,
204204
documentCollection),
@@ -265,10 +265,10 @@ private Mono<IFaultInjectionRuleInternal> getEffectiveConnectionErrorRule(
265265
DocumentCollection documentCollection) {
266266

267267
return Mono.just(rule)
268-
.flatMap(originalRule -> Mono.just(this.getRegionEndpoints(rule.getCondition())))
269-
.flatMap(regionEndpoints -> {
268+
.flatMap(originalRule -> Mono.just(this.getRegionalRoutingContexts(rule.getCondition())))
269+
.flatMap(regionalRoutingContexts -> {
270270
return this.resolvePhysicalAddresses(
271-
regionEndpoints,
271+
regionalRoutingContexts,
272272
rule.getCondition().getEndpoints(),
273273
this.isWriteOnly(rule.getCondition()),
274274
documentCollection)
@@ -281,9 +281,9 @@ private Mono<IFaultInjectionRuleInternal> getEffectiveConnectionErrorRule(
281281

282282
FaultInjectionConnectionErrorResult result = (FaultInjectionConnectionErrorResult) rule.getResult();
283283

284-
List<URI> regionEndpointsWithDefault = new ArrayList<>(regionEndpoints);
284+
List<RegionalRoutingContext> regionEndpointsWithDefault = new ArrayList<>(regionalRoutingContexts);
285285
// if region is not specific configured, then also add the defaultEndpoint
286-
regionEndpointsWithDefault.add(this.globalEndpointManager.getDefaultEndpoint());
286+
regionEndpointsWithDefault.add(new RegionalRoutingContext(this.globalEndpointManager.getDefaultEndpoint()));
287287

288288
return new FaultInjectionConnectionErrorRule(
289289
rule.getId(),
@@ -306,16 +306,16 @@ private Mono<IFaultInjectionRuleInternal> getEffectiveConnectionErrorRule(
306306
* @param condition the fault injection condition.
307307
* @return the region service endpoints.
308308
*/
309-
private List<URI> getRegionEndpoints(FaultInjectionCondition condition) {
309+
private List<RegionalRoutingContext> getRegionalRoutingContexts(FaultInjectionCondition condition) {
310310
boolean isWriteOnlyEndpoints = this.isWriteOnly(condition);
311311

312312
if (StringUtils.isNotEmpty(condition.getRegion())) {
313313
return Arrays.asList(
314314
this.globalEndpointManager.resolveFaultInjectionServiceEndpoint(condition.getRegion(), isWriteOnlyEndpoints));
315315
} else {
316316
return isWriteOnlyEndpoints
317-
? this.globalEndpointManager.getAvailableWriteEndpoints()
318-
: this.globalEndpointManager.getAvailableReadEndpoints();
317+
? this.globalEndpointManager.getAvailableWriteRoutingContexts()
318+
: this.globalEndpointManager.getAvailableReadRoutingContexts();
319319
}
320320
}
321321

@@ -410,7 +410,7 @@ private Mono<List<String>> resolvePartitionKeyRangeIds(
410410

411411

412412
private Mono<List<URI>> resolvePhysicalAddresses(
413-
List<URI> regionEndpoints,
413+
List<RegionalRoutingContext> regionalRoutingContexts,
414414
FaultInjectionEndpoints addressEndpoints,
415415
boolean isWriteOnly,
416416
DocumentCollection documentCollection) {
@@ -419,8 +419,8 @@ private Mono<List<URI>> resolvePhysicalAddresses(
419419
return Mono.just(Arrays.asList());
420420
}
421421

422-
return Flux.fromIterable(regionEndpoints)
423-
.flatMap(regionEndpoint -> {
422+
return Flux.fromIterable(regionalRoutingContexts)
423+
.flatMap(regionalRoutingContext -> {
424424
FeedRangeInternal feedRangeInternal = FeedRangeInternal.convert(addressEndpoints.getFeedRange());
425425
RxDocumentServiceRequest request = RxDocumentServiceRequest.create(
426426
null,
@@ -447,7 +447,7 @@ private Mono<List<URI>> resolvePhysicalAddresses(
447447
ResourceType.Document,
448448
null);
449449

450-
faultInjectionAddressRequest.requestContext.regionalRoutingContextToRoute = new RegionalRoutingContext(regionEndpoint);
450+
faultInjectionAddressRequest.requestContext.regionalRoutingContextToRoute = regionalRoutingContext;
451451
faultInjectionAddressRequest.setPartitionKeyRangeIdentity(new PartitionKeyRangeIdentity(pkRangeId));
452452

453453
if (isWriteOnly) {

0 commit comments

Comments
 (0)