Skip to content

Commit 0b23b9a

Browse files
suresh-prakashsuresh-prakash
authored
Handling duplicate selections in MongoDB (#66)
* Handle duplications using (i) a overwrite strategy for duplicate selections (ii) an error-out strategy for duplicate aliases
1 parent 506300c commit 0b23b9a

File tree

4 files changed

+178
-3
lines changed

4 files changed

+178
-3
lines changed

‎document-store/build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ dependencies {
1313
implementation("org.apache.commons:commons-collections4:4.4")
1414
implementation("org.postgresql:postgresql:42.2.13")
1515
implementation("org.mongodb:mongodb-driver-sync:4.1.2")
16-
implementation("com.fasterxml.jackson.core:jackson-databind:2.11.0")
16+
implementation("com.fasterxml.jackson.core:jackson-databind:2.12.6")
1717
implementation("org.slf4j:slf4j-api:1.7.32")
1818
implementation("com.google.guava:guava-annotations:r03")
1919
implementation("org.apache.commons:commons-lang3:3.10")

‎document-store/src/integrationTest/java/org/hypertrace/core/documentstore/mongo/MongoQueryExecutorIntegrationTest.java‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,32 @@ public void testFindSimple() throws IOException {
127127
assertDocsEqual(resultDocs, "mongo/simple_filter_response.json");
128128
}
129129

130+
@Test
131+
public void testFindWithDuplicateSelections() throws IOException {
132+
List<SelectionSpec> selectionSpecs =
133+
List.of(
134+
SelectionSpec.of(IdentifierExpression.of("item")),
135+
SelectionSpec.of(IdentifierExpression.of("item")),
136+
SelectionSpec.of(IdentifierExpression.of("price")),
137+
SelectionSpec.of(IdentifierExpression.of("quantity")),
138+
SelectionSpec.of(IdentifierExpression.of("quantity")),
139+
SelectionSpec.of(IdentifierExpression.of("date")));
140+
Selection selection = Selection.builder().selectionSpecs(selectionSpecs).build();
141+
Filter filter =
142+
Filter.builder()
143+
.expression(
144+
RelationalExpression.of(
145+
IdentifierExpression.of("item"),
146+
NOT_IN,
147+
ConstantExpression.ofStrings(List.of("Soap", "Bottle"))))
148+
.build();
149+
150+
Query query = Query.builder().setSelection(selection).setFilter(filter).build();
151+
152+
Iterator<Document> resultDocs = collection.find(query);
153+
assertDocsEqual(resultDocs, "mongo/simple_filter_response.json");
154+
}
155+
130156
@Test
131157
public void testFindWithSortingAndPagination() throws IOException {
132158
List<SelectionSpec> selectionSpecs =
@@ -166,6 +192,47 @@ public void testFindWithSortingAndPagination() throws IOException {
166192
assertDocsEqual(resultDocs, "mongo/filter_with_sorting_and_pagination_response.json");
167193
}
168194

195+
@Test
196+
public void testFindWithDuplicateSortingAndPagination() throws IOException {
197+
List<SelectionSpec> selectionSpecs =
198+
List.of(
199+
SelectionSpec.of(IdentifierExpression.of("item")),
200+
SelectionSpec.of(IdentifierExpression.of("price")),
201+
SelectionSpec.of(IdentifierExpression.of("quantity")),
202+
SelectionSpec.of(IdentifierExpression.of("date")));
203+
Selection selection = Selection.builder().selectionSpecs(selectionSpecs).build();
204+
205+
Filter filter =
206+
Filter.builder()
207+
.expression(
208+
RelationalExpression.of(
209+
IdentifierExpression.of("item"),
210+
IN,
211+
ConstantExpression.ofStrings(List.of("Mirror", "Comb", "Shampoo", "Bottle"))))
212+
.build();
213+
214+
Sort sort =
215+
Sort.builder()
216+
.sortingSpec(SortingSpec.of(IdentifierExpression.of("quantity"), DESC))
217+
.sortingSpec(SortingSpec.of(IdentifierExpression.of("item"), ASC))
218+
.sortingSpec(SortingSpec.of(IdentifierExpression.of("quantity"), DESC))
219+
.sortingSpec(SortingSpec.of(IdentifierExpression.of("item"), ASC))
220+
.build();
221+
222+
Pagination pagination = Pagination.builder().offset(1).limit(3).build();
223+
224+
Query query =
225+
Query.builder()
226+
.setSelection(selection)
227+
.setFilter(filter)
228+
.setSort(sort)
229+
.setPagination(pagination)
230+
.build();
231+
232+
Iterator<Document> resultDocs = collection.find(query);
233+
assertDocsEqual(resultDocs, "mongo/filter_with_sorting_and_pagination_response.json");
234+
}
235+
169236
@Test
170237
public void testFindWithNestedFields() throws IOException {
171238
List<SelectionSpec> selectionSpecs =
@@ -221,6 +288,18 @@ public void testAggregateSimple() throws IOException {
221288
assertDocsEqual(resultDocs, "mongo/count_response.json");
222289
}
223290

291+
@Test
292+
public void testAggregateWithDuplicateSelections() throws IOException {
293+
Query query =
294+
Query.builder()
295+
.addSelection(AggregateExpression.of(COUNT, IdentifierExpression.of("item")), "count")
296+
.addSelection(AggregateExpression.of(COUNT, IdentifierExpression.of("item")), "count")
297+
.build();
298+
299+
Iterator<Document> resultDocs = collection.aggregate(query);
300+
assertDocsEqual(resultDocs, "mongo/count_response.json");
301+
}
302+
224303
@Test
225304
public void testAggregateWithFiltersAndOrdering() throws IOException {
226305
Query query =
@@ -257,6 +336,45 @@ public void testAggregateWithFiltersAndOrdering() throws IOException {
257336
assertDocsEqual(resultDocs, "mongo/sum_response.json");
258337
}
259338

339+
@Test
340+
public void testAggregateWithFiltersAndDuplicateOrderingAndDuplicateAggregations()
341+
throws IOException {
342+
Query query =
343+
Query.builder()
344+
.addSelection(
345+
AggregateExpression.of(
346+
SUM,
347+
FunctionExpression.builder()
348+
.operand(IdentifierExpression.of("price"))
349+
.operator(MULTIPLY)
350+
.operand(IdentifierExpression.of("quantity"))
351+
.build()),
352+
"total")
353+
.addSelection(IdentifierExpression.of("item"))
354+
.addAggregation(IdentifierExpression.of("item"))
355+
.addAggregation(IdentifierExpression.of("item"))
356+
.addSort(IdentifierExpression.of("total"), DESC)
357+
.addSort(IdentifierExpression.of("total"), DESC)
358+
.setAggregationFilter(
359+
LogicalExpression.builder()
360+
.operand(
361+
RelationalExpression.of(
362+
IdentifierExpression.of("total"), GTE, ConstantExpression.of(11)))
363+
.operator(AND)
364+
.operand(
365+
RelationalExpression.of(
366+
IdentifierExpression.of("total"), LTE, ConstantExpression.of(99)))
367+
.build())
368+
.setFilter(
369+
RelationalExpression.of(
370+
IdentifierExpression.of("quantity"), NEQ, ConstantExpression.of(10)))
371+
.setPagination(Pagination.builder().limit(10).offset(0).build())
372+
.build();
373+
374+
Iterator<Document> resultDocs = collection.aggregate(query);
375+
assertDocsEqual(resultDocs, "mongo/sum_response.json");
376+
}
377+
260378
@Test
261379
public void testAggregateWithNestedFields() throws IOException {
262380
Query query =

‎document-store/src/main/java/org/hypertrace/core/documentstore/mongo/parser/MongoSelectingExpressionParser.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,11 @@ public static BasicDBObject getSelections(final Query query) {
5757
selectionSpecs.stream()
5858
.map(spec -> MongoSelectingExpressionParser.parse(parser, spec))
5959
.flatMap(map -> map.entrySet().stream())
60-
.collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
60+
.collect(
61+
toMap(
62+
Map.Entry::getKey,
63+
Map.Entry::getValue,
64+
MongoSelectingExpressionParser::mergeValues));
6165

6266
return new BasicDBObject(projectionMap);
6367
}
@@ -79,4 +83,14 @@ private static Map<String, Object> parse(
7983

8084
return spec.getExpression().visit(parser);
8185
}
86+
87+
private static <T> T mergeValues(final T first, final T second) {
88+
if (first.equals(second)) {
89+
return second;
90+
}
91+
92+
throw new IllegalArgumentException(
93+
String.format(
94+
"Query contains duplicate aliases with different selections: (%s, %s)", first, second));
95+
}
8296
}

‎document-store/src/test/java/org/hypertrace/core/documentstore/mongo/MongoQueryExecutorTest.java‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import static org.hypertrace.core.documentstore.expression.operators.RelationalOperator.NOT_IN;
1818
import static org.hypertrace.core.documentstore.expression.operators.SortingOrder.ASC;
1919
import static org.hypertrace.core.documentstore.expression.operators.SortingOrder.DESC;
20+
import static org.junit.jupiter.api.Assertions.assertThrows;
2021
import static org.mockito.ArgumentMatchers.any;
2122
import static org.mockito.ArgumentMatchers.anyInt;
2223
import static org.mockito.ArgumentMatchers.anyList;
@@ -37,8 +38,10 @@
3738
import org.hypertrace.core.documentstore.expression.impl.LogicalExpression;
3839
import org.hypertrace.core.documentstore.expression.impl.RelationalExpression;
3940
import org.hypertrace.core.documentstore.expression.operators.SortingOrder;
41+
import org.hypertrace.core.documentstore.query.Filter;
4042
import org.hypertrace.core.documentstore.query.Pagination;
4143
import org.hypertrace.core.documentstore.query.Query;
44+
import org.hypertrace.core.documentstore.query.Selection;
4245
import org.hypertrace.core.documentstore.query.SelectionSpec;
4346
import org.hypertrace.core.documentstore.query.SortingSpec;
4447
import org.junit.jupiter.api.AfterEach;
@@ -85,7 +88,6 @@ void setUp() {
8588

8689
@AfterEach
8790
void tearDown() {
88-
verify(collection).getNamespace();
8991
verifyNoMoreInteractions(collection, iterable, cursor, aggIterable);
9092
}
9193

@@ -98,6 +100,7 @@ public void testFindSimple() {
98100
BasicDBObject mongoQuery = new BasicDBObject();
99101
BasicDBObject projection = new BasicDBObject();
100102

103+
verify(collection).getNamespace();
101104
verify(collection).find(mongoQuery);
102105
verify(iterable).projection(projection);
103106
verify(iterable, NOT_INVOKED).sort(any());
@@ -119,6 +122,7 @@ public void testFindWithSelection() {
119122
BasicDBObject mongoQuery = new BasicDBObject();
120123
BasicDBObject projection = BasicDBObject.parse("{id: 1, name: \"$fname\"}");
121124

125+
verify(collection).getNamespace();
122126
verify(collection).find(mongoQuery);
123127
verify(iterable).projection(projection);
124128
verify(iterable, NOT_INVOKED).sort(any());
@@ -159,6 +163,7 @@ public void testFindWithFilter() {
159163
+ "}");
160164
BasicDBObject projection = new BasicDBObject();
161165

166+
verify(collection).getNamespace();
162167
verify(collection).find(mongoQuery);
163168
verify(iterable).projection(projection);
164169
verify(iterable, NOT_INVOKED).sort(any());
@@ -181,6 +186,7 @@ public void testFindWithSorting() {
181186
BasicDBObject sortQuery = BasicDBObject.parse("{ marks: -1, name: 1}");
182187
BasicDBObject projection = new BasicDBObject();
183188

189+
verify(collection).getNamespace();
184190
verify(collection).find(mongoQuery);
185191
verify(iterable).projection(projection);
186192
verify(iterable).sort(sortQuery);
@@ -199,6 +205,7 @@ public void testFindWithPagination() {
199205
BasicDBObject mongoQuery = new BasicDBObject();
200206
BasicDBObject projection = new BasicDBObject();
201207

208+
verify(collection).getNamespace();
202209
verify(collection).find(mongoQuery);
203210
verify(iterable).projection(projection);
204211
verify(iterable, NOT_INVOKED).sort(any());
@@ -245,6 +252,7 @@ public void testFindWithAllClauses() {
245252
BasicDBObject projection = BasicDBObject.parse("{id: 1, name: \"$fname\"}");
246253
BasicDBObject sortQuery = BasicDBObject.parse("{ marks: -1, name: 1}");
247254

255+
verify(collection).getNamespace();
248256
verify(collection).find(mongoQuery);
249257
verify(iterable).projection(projection);
250258
verify(iterable).sort(sortQuery);
@@ -253,6 +261,40 @@ public void testFindWithAllClauses() {
253261
verify(iterable).cursor();
254262
}
255263

264+
@Test
265+
public void testFindAndAggregateWithDuplicateAlias() {
266+
List<SelectionSpec> selectionSpecs =
267+
List.of(
268+
SelectionSpec.of(IdentifierExpression.of("item")),
269+
SelectionSpec.of(IdentifierExpression.of("price"), "value"),
270+
SelectionSpec.of(IdentifierExpression.of("quantity"), "value"),
271+
SelectionSpec.of(IdentifierExpression.of("date")));
272+
Selection selection = Selection.builder().selectionSpecs(selectionSpecs).build();
273+
Filter filter =
274+
Filter.builder()
275+
.expression(
276+
RelationalExpression.of(
277+
IdentifierExpression.of("item"),
278+
NOT_IN,
279+
ConstantExpression.ofStrings(List.of("Soap", "Bottle"))))
280+
.build();
281+
282+
Query query = Query.builder().setSelection(selection).setFilter(filter).build();
283+
284+
assertThrows(IllegalArgumentException.class, () -> executor.find(query));
285+
verify(collection, NOT_INVOKED).getNamespace();
286+
verify(collection, NOT_INVOKED).find(any(BasicDBObject.class));
287+
verify(iterable, NOT_INVOKED).projection(any(BasicDBObject.class));
288+
verify(iterable, NOT_INVOKED).sort(any(BasicDBObject.class));
289+
verify(iterable, NOT_INVOKED).skip(anyInt());
290+
verify(iterable, NOT_INVOKED).limit(anyInt());
291+
verify(iterable, NOT_INVOKED).cursor();
292+
293+
assertThrows(IllegalArgumentException.class, () -> executor.aggregate(query));
294+
verify(collection, NOT_INVOKED).aggregate(anyList());
295+
verify(aggIterable, NOT_INVOKED).cursor();
296+
}
297+
256298
@Test
257299
public void testSimpleAggregate() {
258300
Query query =
@@ -549,6 +591,7 @@ public void testGetDistinctCount() {
549591

550592
private void testAggregation(Query query, List<BasicDBObject> pipeline) {
551593
executor.aggregate(query);
594+
verify(collection).getNamespace();
552595
verify(collection).aggregate(pipeline);
553596
verify(aggIterable).cursor();
554597
}

0 commit comments

Comments
 (0)