Skip to content

Commit 5499f82

Browse files
SrinikhilReddymoderakh
SrinikhilReddy
and
moderakh
authored
Subpartitioning: Adds SDK changes to support subpartitioning. (Azure#18503)
* Changes to run tests for multihash collections * Changes. * Parititionkey builder model as suggested by Mo. * Apply suggestions from code review Co-authored-by: Mohammad Derakhshani <[email protected]> * Pushing to remote. Cleanup * commit local changes * Address PR comments. * Address code comments. * Resolve code comments * Add tests. * Well, when does this stop. * No more comments. * Update CosmosMultiHashTest.java Updating a test.. * Fix test failures * Fixing code suggestions * Resolve code comments * Fixing a test Co-authored-by: Mohammad Derakhshani <[email protected]>
1 parent bf5f172 commit 5499f82

File tree

9 files changed

+472
-16
lines changed

9 files changed

+472
-16
lines changed

‎sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/HttpConstants.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ public static class A_IMHeaderValues {
273273
}
274274

275275
public static class Versions {
276-
public static final String CURRENT_VERSION = "2018-12-31";
276+
public static final String CURRENT_VERSION = "2020-07-15";
277277
public static final String QUERY_VERSION = "1.0";
278278
public static final String AZURE_COSMOS_PROPERTIES_FILE_NAME = "azure-cosmos.properties";
279279

‎sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1310,20 +1310,35 @@ public static PartitionKeyInternal extractPartitionKeyValueFromDocument(
13101310
InternalObjectNode document,
13111311
PartitionKeyDefinition partitionKeyDefinition) {
13121312
if (partitionKeyDefinition != null) {
1313-
String path = partitionKeyDefinition.getPaths().iterator().next();
1314-
List<String> parts = PathParser.getPathParts(path);
1315-
if (parts.size() >= 1) {
1316-
Object value = ModelBridgeInternal.getObjectByPathFromJsonSerializable(document, parts);
1317-
if (value == null || value.getClass() == ObjectNode.class) {
1318-
value = ModelBridgeInternal.getNonePartitionKey(partitionKeyDefinition);
1319-
}
1313+
switch (partitionKeyDefinition.getKind()) {
1314+
case HASH:
1315+
String path = partitionKeyDefinition.getPaths().iterator().next();
1316+
List<String> parts = PathParser.getPathParts(path);
1317+
if (parts.size() >= 1) {
1318+
Object value = ModelBridgeInternal.getObjectByPathFromJsonSerializable(document, parts);
1319+
if (value == null || value.getClass() == ObjectNode.class) {
1320+
value = ModelBridgeInternal.getNonePartitionKey(partitionKeyDefinition);
1321+
}
13201322

1321-
if (value instanceof PartitionKeyInternal) {
1322-
return (PartitionKeyInternal) value;
1323-
} else {
1324-
return PartitionKeyInternal.fromObjectArray(Collections.singletonList(value), false);
1323+
if (value instanceof PartitionKeyInternal) {
1324+
return (PartitionKeyInternal) value;
1325+
} else {
1326+
return PartitionKeyInternal.fromObjectArray(Collections.singletonList(value), false);
1327+
}
1328+
}
1329+
break;
1330+
case MULTI_HASH:
1331+
Object[] partitionKeyValues = new Object[partitionKeyDefinition.getPaths().size()];
1332+
for(int pathIter = 0 ; pathIter < partitionKeyDefinition.getPaths().size(); pathIter++){
1333+
String partitionPath = partitionKeyDefinition.getPaths().get(pathIter);
1334+
List<String> partitionPathParts = PathParser.getPathParts(partitionPath);
1335+
partitionKeyValues[pathIter] = ModelBridgeInternal.getObjectByPathFromJsonSerializable(document, partitionPathParts);
1336+
}
1337+
return PartitionKeyInternal.fromObjectArray(partitionKeyValues, false);
1338+
1339+
default:
1340+
throw new IllegalArgumentException("Unrecognized Partition kind: " + partitionKeyDefinition.getKind());
13251341
}
1326-
}
13271342
}
13281343

13291344
return null;

‎sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/routing/PartitionKeyInternalHelper.java

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ public class PartitionKeyInternalHelper {
3535
(byte) 0x3F, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF,
3636
(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF});
3737

38+
private static final int HASH_V2_EPK_LENGTH = 32;
39+
3840
static byte[] uIntToBytes(UInt128 unit) {
3941
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * 2);
4042
buffer.putLong(unit.low);
@@ -92,6 +94,31 @@ static public String getEffectivePartitionKeyForHashPartitioningV2(PartitionKeyI
9294
}
9395
}
9496

97+
static String getEffectivePartitionKeyForMultiHashPartitioning(PartitionKeyInternal partitionKeyInternal) {
98+
StringBuilder stringBuilder = new StringBuilder(partitionKeyInternal.components.size() * HASH_V2_EPK_LENGTH);
99+
for (int i = 0; i < partitionKeyInternal.components.size(); i++) {
100+
try(ByteBufferOutputStream byteArrayBuffer = new ByteBufferOutputStream()) {
101+
partitionKeyInternal.components.get(i).writeForHashingV2(byteArrayBuffer);
102+
103+
ByteBuffer byteBuffer = byteArrayBuffer.asByteBuffer();
104+
UInt128 hashAsUnit128 = MurmurHash3_128.hash128(byteBuffer.array(), byteBuffer.limit());
105+
106+
byte[] hash = uIntToBytes(hashAsUnit128);
107+
Bytes.reverse(hash);
108+
109+
// Reset 2 most significant bits, as max exclusive value is 'FF'.
110+
// Plus one more just in case.
111+
hash[0] &= 0x3F;
112+
113+
stringBuilder.append(HexConvert.bytesToHex(hash));
114+
} catch (IOException e) {
115+
throw new IllegalArgumentException(e);
116+
}
117+
}
118+
119+
return stringBuilder.toString();
120+
}
121+
95122
static String getEffectivePartitionKeyForHashPartitioning(PartitionKeyInternal partitionKeyInternal) {
96123
IPartitionKeyComponent[] truncatedComponents = new IPartitionKeyComponent[partitionKeyInternal.components.size()];
97124

@@ -138,7 +165,7 @@ public static String getEffectivePartitionKeyString(PartitionKeyInternal partiti
138165
return MaximumExclusiveEffectivePartitionKey;
139166
}
140167

141-
if (partitionKeyInternal.components.size() < partitionKeyDefinition.getPaths().size()) {
168+
if (partitionKeyInternal.components.size() < partitionKeyDefinition.getPaths().size() && partitionKeyDefinition.getKind() != PartitionKind.MULTI_HASH) {
142169
throw new IllegalArgumentException(RMResources.TooFewPartitionKeyComponents);
143170
}
144171

@@ -161,6 +188,9 @@ public static String getEffectivePartitionKeyString(PartitionKeyInternal partiti
161188
return getEffectivePartitionKeyForHashPartitioning(partitionKeyInternal);
162189
}
163190

191+
case MULTI_HASH:
192+
return getEffectivePartitionKeyForMultiHashPartitioning(partitionKeyInternal);
193+
164194
default:
165195
return toHexEncodedBinaryString(partitionKeyInternal.components);
166196
}

‎sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/PartitionKeyBuilder.java

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
package com.azure.cosmos.models;
5+
6+
import com.azure.cosmos.implementation.Undefined;
7+
import com.azure.cosmos.implementation.routing.PartitionKeyInternal;
8+
import com.azure.cosmos.util.Beta;
9+
import com.azure.cosmos.util.Beta.SinceVersion;
10+
11+
import java.util.ArrayList;
12+
import java.util.List;
13+
14+
@Beta(value = SinceVersion.V4_16_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING)
15+
public final class PartitionKeyBuilder {
16+
private final List<Object> partitionKeyValues;
17+
18+
/**
19+
* Constructor. CREATE a new instance of the PartitionKeyBuilder object.
20+
*/
21+
@Beta(value = SinceVersion.V4_16_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING)
22+
public PartitionKeyBuilder() {
23+
this.partitionKeyValues = new ArrayList<Object>();
24+
}
25+
26+
/**
27+
* Adds partition value of type string
28+
* @param value The value of type string to be used as partition key
29+
* @return The current PartitionKeyBuilder object
30+
*/
31+
@Beta(value = SinceVersion.V4_16_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING)
32+
public PartitionKeyBuilder add(String value) {
33+
this.partitionKeyValues.add(value);
34+
return this;
35+
}
36+
37+
/**
38+
* Adds partition value of type double
39+
* @param value The value of type double to be used as partition key
40+
* @return The current PartitionKeyBuilder object
41+
*/
42+
@Beta(value = SinceVersion.V4_16_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING)
43+
public PartitionKeyBuilder add(double value) {
44+
this.partitionKeyValues.add(value);
45+
return this;
46+
}
47+
48+
/**
49+
* Adds partition value of type boolean
50+
* @param value The value of type boolean to be used as partition key
51+
* @return The current PartitionKeyBuilder object
52+
*/
53+
@Beta(value = SinceVersion.V4_16_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING)
54+
public PartitionKeyBuilder add(boolean value) {
55+
this.partitionKeyValues.add(value);
56+
return this;
57+
}
58+
59+
/**
60+
* Adds a null partition key value
61+
* @return The current PartitionKeyBuilder object
62+
*/
63+
@Beta(value = SinceVersion.V4_16_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING)
64+
public PartitionKeyBuilder addNullValue() {
65+
this.partitionKeyValues.add(null);
66+
return this;
67+
}
68+
69+
/**
70+
* Adds a None Partition Key
71+
* @return The current PartitionKeyBuilder object
72+
*/
73+
@Beta(value = SinceVersion.V4_16_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING)
74+
public PartitionKeyBuilder addNoneValue() {
75+
this.partitionKeyValues.add(PartitionKey.NONE);
76+
return this;
77+
}
78+
79+
/**
80+
* Builds a new instance of the type PartitionKey with the specified Partition Key values.
81+
* @return PartitionKey object
82+
*/
83+
@Beta(value = SinceVersion.V4_16_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING)
84+
public PartitionKey build() {
85+
// Why these checks?
86+
// These changes are being added for SDK to support multiple paths in a partition key.
87+
//
88+
// Currently, when a resource does not specify a value for the PartitionKey,
89+
// we assign a temporary value `PartitionKey.None` and later discern whether
90+
// it is a PartitionKey.Undefined or PartitionKey.Empty based on the Collection Type.
91+
// We retain this behaviour for single path partition keys.
92+
//
93+
// For collections with multiple path keys, absence of a partition key values is
94+
// always treated as a PartitionKey.Undefined.
95+
if(this.partitionKeyValues.size() == 0) {
96+
throw new IllegalArgumentException("No partition key value has been specified");
97+
}
98+
99+
if(this.partitionKeyValues.size() == 1 && PartitionKey.NONE.equals(this.partitionKeyValues.get(0))) {
100+
return PartitionKey.NONE;
101+
}
102+
103+
PartitionKeyInternal partitionKeyInternal;
104+
Object[] valueArray = new Object[this.partitionKeyValues.size()];
105+
for(int i = 0; i < this.partitionKeyValues.size(); i++) {
106+
Object val = this.partitionKeyValues.get(i);
107+
if(PartitionKey.NONE.equals(val)) {
108+
valueArray[i] = Undefined.value();
109+
}
110+
else {
111+
valueArray[i] = val;
112+
}
113+
}
114+
115+
partitionKeyInternal = PartitionKeyInternal.fromObjectArray(valueArray, true);
116+
return new PartitionKey(partitionKeyInternal);
117+
}
118+
}

‎sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/PartitionKind.java

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,24 @@
33

44
package com.azure.cosmos.models;
55

6+
import com.azure.cosmos.util.Beta;
7+
import com.azure.cosmos.util.Beta.SinceVersion;
8+
69
/**
710
* Specifies the partition scheme for an multiple-partitioned container in the Azure Cosmos DB database service.
811
*/
912
public enum PartitionKind {
1013
/**
1114
* The Partition of a item is calculated based on the hash value of the PartitionKey.
1215
*/
13-
HASH("Hash");
16+
HASH("Hash"),
17+
18+
RANGE("Range"),
19+
/**
20+
* The Partition of a item is calculated based on the hash value of multiple PartitionKeys.
21+
*/
22+
@Beta(value = SinceVersion.V4_16_0, warningText = Beta.PREVIEW_SUBJECT_TO_CHANGE_WARNING)
23+
MULTI_HASH("MultiHash");
1424

1525
PartitionKind(String overWireValue) {
1626
this.overWireValue = overWireValue;

‎sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/util/Beta.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,14 @@
1010
import java.lang.annotation.Target;
1111

1212
import static java.lang.annotation.ElementType.CONSTRUCTOR;
13+
import static java.lang.annotation.ElementType.FIELD;
1314
import static java.lang.annotation.ElementType.METHOD;
1415
import static java.lang.annotation.ElementType.PARAMETER;
1516
import static java.lang.annotation.ElementType.TYPE;
1617

1718
@Documented
1819
@Retention(RetentionPolicy.CLASS)
19-
@Target({ TYPE, METHOD, PARAMETER, CONSTRUCTOR })
20+
@Target({ TYPE, METHOD, PARAMETER, CONSTRUCTOR, FIELD })
2021
@Inherited
2122
/**
2223
* Indicates functionality that is in preview and as such is subject to change in non-backwards compatible ways in future releases,

‎sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosContainerTest.java

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
import com.azure.cosmos.models.IndexingPolicy;
2828
import com.azure.cosmos.models.PartitionKey;
2929
import com.azure.cosmos.models.PartitionKeyDefinition;
30+
import com.azure.cosmos.models.PartitionKeyDefinitionVersion;
31+
import com.azure.cosmos.models.PartitionKind;
3032
import com.azure.cosmos.models.SqlQuerySpec;
3133
import com.azure.cosmos.models.ThroughputProperties;
3234
import com.azure.cosmos.rx.TestSuiteBase;
@@ -47,6 +49,7 @@
4749
import java.util.Base64;
4850
import java.util.List;
4951
import java.util.UUID;
52+
import java.util.ArrayList;
5053

5154
import static org.assertj.core.api.Assertions.assertThat;
5255
import static org.assertj.core.api.Assertions.fail;
@@ -764,6 +767,40 @@ public void readAllContainers() throws Exception{
764767
assertThat(feedResponseIterator1.iterator().hasNext()).isTrue();
765768
}
766769

770+
@Test(groups = { "emulator" }, timeOut = TIMEOUT)
771+
public void crudMultiHashContainer() throws Exception {
772+
String collectionName = UUID.randomUUID().toString();
773+
774+
PartitionKeyDefinition partitionKeyDefinition = new PartitionKeyDefinition();
775+
partitionKeyDefinition.setKind(PartitionKind.MULTI_HASH);
776+
partitionKeyDefinition.setVersion(PartitionKeyDefinitionVersion.V2);
777+
ArrayList<String> paths = new ArrayList<>();
778+
paths.add("/city");
779+
paths.add("/zipcode");
780+
partitionKeyDefinition.setPaths(paths);
781+
782+
CosmosContainerProperties containerProperties = getCollectionDefinition(collectionName, partitionKeyDefinition);
783+
784+
//MultiHash collection create
785+
CosmosContainerResponse containerResponse = createdDatabase.createContainer(containerProperties);
786+
validateContainerResponse(containerProperties, containerResponse);
787+
assertThat(containerResponse.getProperties().getPartitionKeyDefinition().getKind() == PartitionKind.MULTI_HASH);
788+
assertThat(containerResponse.getProperties().getPartitionKeyDefinition().getPaths().size() == paths.size());
789+
assertThat(containerResponse.getProperties().getPartitionKeyDefinition().getPaths().get(0) == paths.get(0));
790+
assertThat(containerResponse.getProperties().getPartitionKeyDefinition().getPaths().get(1) == paths.get(1));
791+
792+
//MultiHash collection read
793+
CosmosContainer multiHashContainer = createdDatabase.getContainer(collectionName);
794+
containerResponse = multiHashContainer.read();
795+
validateContainerResponse(containerProperties, containerResponse);
796+
assertThat(containerResponse.getProperties().getPartitionKeyDefinition().getKind() == PartitionKind.MULTI_HASH);
797+
assertThat(containerResponse.getProperties().getPartitionKeyDefinition().getPaths().size() == paths.size());
798+
assertThat(containerResponse.getProperties().getPartitionKeyDefinition().getPaths().get(0) == paths.get(0));
799+
assertThat(containerResponse.getProperties().getPartitionKeyDefinition().getPaths().get(1) == paths.get(1));
800+
801+
//MultiHash collection delete
802+
CosmosContainerResponse deleteResponse = multiHashContainer.delete();
803+
}
767804

768805
@Test(groups = { "emulator" }, timeOut = TIMEOUT)
769806
public void queryContainer() throws Exception{

0 commit comments

Comments
 (0)