diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index f2527bb0..0902d7cd 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -80,13 +80,16 @@ import java.util.stream.StreamSupport; import org.hypertrace.core.documentstore.commons.DocStoreConstants; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.KeyExpression; import org.hypertrace.core.documentstore.expression.impl.LogicalExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; +import org.hypertrace.core.documentstore.expression.impl.SubQueryJoinExpression; import org.hypertrace.core.documentstore.expression.impl.UnnestExpression; +import org.hypertrace.core.documentstore.expression.operators.AggregationOperator; import org.hypertrace.core.documentstore.expression.operators.FunctionOperator; import org.hypertrace.core.documentstore.expression.operators.RelationalOperator; import org.hypertrace.core.documentstore.expression.type.FilterTypeExpression; @@ -3478,6 +3481,182 @@ public void testToLowerCaseMongoFunctionOperator(String dataStoreName) throws Ex dataStoreName, resultDocs, "query/case_insensitive_exact_match_response.json", 2); } + @ParameterizedTest + @ArgumentsSource(MongoProvider.class) + void testSelfJoinWithSubQuery(String dataStoreName) throws IOException { + Collection collection = getCollection(dataStoreName); + + /* + This is the query we want to execute: + SELECT item, quantity, date + FROM + JOIN ( + SELECT item, MAX(date) AS latest_date + FROM + GROUP BY item + ) latest + ON item = latest.item + AND date = latest.latest_date + ORDER BY `item` ASC; + */ + + /* + The right subquery: + SELECT item, MAX(date) AS latest_date + FROM + GROUP BY item + */ + Query subQuery = + Query.builder() + .addSelection(SelectionSpec.of(IdentifierExpression.of("item"))) + .addSelection( + SelectionSpec.of( + AggregateExpression.of( + AggregationOperator.MAX, IdentifierExpression.of("date")), + "latest_date")) + .addAggregation(IdentifierExpression.of("item")) + .build(); + + /* + The FROM expression representing a join with the right subquery: + FROM + JOIN ( + SELECT item, MAX(date) AS latest_date + FROM + GROUP BY item + ) latest + ON item = latest.item + AND date = latest.latest_date; + */ + SubQueryJoinExpression subQueryJoinExpression = + SubQueryJoinExpression.builder() + .subQuery(subQuery) + .subQueryAlias("latest") + .joinCondition( + LogicalExpression.and( + RelationalExpression.of( + IdentifierExpression.of("item"), + RelationalOperator.EQ, + AliasedIdentifierExpression.builder() + .name("item") + .contextAlias("latest") + .build()), + RelationalExpression.of( + IdentifierExpression.of("date"), + RelationalOperator.EQ, + AliasedIdentifierExpression.builder() + .name("latest_date") + .contextAlias("latest") + .build()))) + .build(); + + /* + Now build the top-level Query: + SELECT item, quantity, date FROM ORDER BY `item` ASC; + */ + Query mainQuery = + Query.builder() + .addSelection(IdentifierExpression.of("item")) + .addSelection(IdentifierExpression.of("quantity")) + .addSelection(IdentifierExpression.of("date")) + .addFromClause(subQueryJoinExpression) + .addSort(IdentifierExpression.of("item"), ASC) + .build(); + + Iterator iterator = collection.aggregate(mainQuery); + assertDocsAndSizeEqual( + dataStoreName, iterator, "query/self_join_with_sub_query_response.json", 4); + } + + @ParameterizedTest + @ArgumentsSource(MongoProvider.class) + void testSelfJoinWithSubQueryWithNestedFields(String dataStoreName) throws IOException { + createCollectionData( + "query/items_data_with_nested_fields.json", "items_data_with_nested_fields"); + Collection collection = getCollection(dataStoreName, "items_data_with_nested_fields"); + + /* + This is the query we want to execute: + SELECT itemDetails.item, itemDetails.quantity, itemDetails.date + FROM + JOIN ( + SELECT itemDetails.item, MAX(itemDetails.date) AS latest_date + FROM + GROUP BY itemDetails.item + ) latest + ON itemDetails.item = latest.itemDetails.item + AND itemDetails.date = latest.latest_date + ORDER BY `itemDetails.item` ASC; + */ + + /* + The right subquery: + SELECT itemDetails.item, MAX(itemDetails.date) AS latest_date + FROM + GROUP BY itemDetails.item + */ + Query subQuery = + Query.builder() + .addSelection(SelectionSpec.of(IdentifierExpression.of("itemDetails.item"))) + .addSelection( + SelectionSpec.of( + AggregateExpression.of( + AggregationOperator.MAX, IdentifierExpression.of("itemDetails.date")), + "latest_date")) + .addAggregation(IdentifierExpression.of("itemDetails.item")) + .build(); + + /* + The FROM expression representing a join with the right subquery: + FROM + JOIN ( + SELECT itemDetails.item, MAX(itemDetails.date) AS latest_date + FROM + GROUP BY itemDetails.item + ) latest + ON itemDetails.item = latest.itemDetails.item + AND itemDetails.date = latest.latest_date; + */ + SubQueryJoinExpression subQueryJoinExpression = + SubQueryJoinExpression.builder() + .subQuery(subQuery) + .subQueryAlias("latest") + .joinCondition( + LogicalExpression.and( + RelationalExpression.of( + IdentifierExpression.of("itemDetails.item"), + RelationalOperator.EQ, + AliasedIdentifierExpression.builder() + .name("itemDetails.item") + .contextAlias("latest") + .build()), + RelationalExpression.of( + IdentifierExpression.of("itemDetails.date"), + RelationalOperator.EQ, + AliasedIdentifierExpression.builder() + .name("latest_date") + .contextAlias("latest") + .build()))) + .build(); + + /* + Now build the top-level Query: + SELECT itemDetails.item, itemDetails.quantity, itemDetails.date FROM ORDER BY `itemDetails.item` ASC; + */ + Query mainQuery = + Query.builder() + .addSelection(IdentifierExpression.of("itemDetails.item")) + .addSelection(IdentifierExpression.of("itemDetails.quantity")) + .addSelection(IdentifierExpression.of("itemDetails.date")) + .addFromClause(subQueryJoinExpression) + .addSort(IdentifierExpression.of("itemDetails.item"), ASC) + .build(); + + Iterator iterator = collection.aggregate(mainQuery); + assertDocsAndSizeEqual( + dataStoreName, iterator, "query/sub_query_join_response_with_nested_fields.json", 3); + } + private static Collection getCollection(final String dataStoreName) { return getCollection(dataStoreName, COLLECTION_NAME); } diff --git a/document-store/src/integrationTest/resources/query/items_data_with_nested_fields.json b/document-store/src/integrationTest/resources/query/items_data_with_nested_fields.json new file mode 100644 index 00000000..4fc28ad7 --- /dev/null +++ b/document-store/src/integrationTest/resources/query/items_data_with_nested_fields.json @@ -0,0 +1,50 @@ +[ + { + "_id": 1, + "itemDetails": { + "item": "Comb", + "date": "2012-01-01", + "quantity": 10 + } + }, + { + "_id": 2, + "itemDetails": { + "item": "Shampoo", + "date": "2012-01-01", + "quantity": 10 + } + }, + { + "_id": 3, + "itemDetails": { + "item": "Shampoo", + "date": "2012-02-02", + "quantity": 20 + } + }, + { + "_id": 4, + "itemDetails": { + "item": "Shampoo", + "date": "2012-03-03", + "quantity": 30 + } + }, + { + "_id": 5, + "itemDetails": { + "item": "Soap", + "date": "2012-02-02", + "quantity": 20 + } + }, + { + "_id": 6, + "itemDetails": { + "item": "Soap", + "date": "2012-01-01", + "quantity": 10 + } + } +] diff --git a/document-store/src/integrationTest/resources/query/self_join_with_sub_query_response.json b/document-store/src/integrationTest/resources/query/self_join_with_sub_query_response.json new file mode 100644 index 00000000..f6862641 --- /dev/null +++ b/document-store/src/integrationTest/resources/query/self_join_with_sub_query_response.json @@ -0,0 +1,22 @@ +[ + { + "date": "2015-09-10T08:43:00Z", + "item": "Comb", + "quantity": 10 + }, + { + "date": "2014-03-01T09:00:00Z", + "item": "Mirror", + "quantity": 1 + }, + { + "date": "2014-04-04T11:21:39.736Z", + "item": "Shampoo", + "quantity": 20 + }, + { + "date": "2016-02-06T20:20:13Z", + "item": "Soap", + "quantity": 5 + } +] diff --git a/document-store/src/integrationTest/resources/query/sub_query_join_response_with_nested_fields.json b/document-store/src/integrationTest/resources/query/sub_query_join_response_with_nested_fields.json new file mode 100644 index 00000000..a6e2774a --- /dev/null +++ b/document-store/src/integrationTest/resources/query/sub_query_join_response_with_nested_fields.json @@ -0,0 +1,23 @@ +[ + { + "itemDetails": { + "item": "Comb", + "date": "2012-01-01", + "quantity": 10 + } + }, + { + "itemDetails": { + "item": "Shampoo", + "date": "2012-03-03", + "quantity": 30 + } + }, + { + "itemDetails": { + "item": "Soap", + "date": "2012-02-02", + "quantity": 20 + } + } +] diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/AliasedIdentifierExpression.java b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/AliasedIdentifierExpression.java new file mode 100644 index 00000000..78e9269e --- /dev/null +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/AliasedIdentifierExpression.java @@ -0,0 +1,69 @@ +package org.hypertrace.core.documentstore.expression.impl; + +import com.google.common.base.Preconditions; +import lombok.Value; +import org.hypertrace.core.documentstore.parser.SelectTypeExpressionVisitor; + +/** + * Expression for referencing an identifier/column name within a context having an alias. + * + *

Example: In this query: + * SELECT item, quantity, date + * FROM + * JOIN ( + * SELECT item, MAX(date) AS latest_date + * FROM + * GROUP BY item + * ) AS latest + * ON item = latest.item + * ORDER BY `item` ASC; + * the rhs of the join condition "latest.item" can be expressed as: + * AliasedIdentifierExpression.builder().name("item").alias("alias1").build() + */ +@Value +public class AliasedIdentifierExpression extends IdentifierExpression { + String contextAlias; + + private AliasedIdentifierExpression(final String name, final String contextAlias) { + super(name); + this.contextAlias = contextAlias; + } + + @Override + public T accept(final SelectTypeExpressionVisitor visitor) { + return visitor.visit(this); + } + + @Override + public String toString() { + return "`" + getContextAlias() + "." + getName() + "`"; + } + + public static AliasedIdentifierExpressionBuilder builder() { + return new AliasedIdentifierExpressionBuilder(); + } + + public static class AliasedIdentifierExpressionBuilder { + private String name; + private String contextAlias; + + public AliasedIdentifierExpressionBuilder name(final String name) { + this.name = name; + return this; + } + + public AliasedIdentifierExpressionBuilder contextAlias(final String contextAlias) { + this.contextAlias = contextAlias; + return this; + } + + public AliasedIdentifierExpression build() { + Preconditions.checkArgument( + this.name != null && !this.name.isBlank(), "name is null or blank"); + Preconditions.checkArgument( + this.contextAlias != null && !this.contextAlias.isBlank(), + "contextAlias is null or blank"); + return new AliasedIdentifierExpression(this.name, this.contextAlias); + } + } +} diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/IdentifierExpression.java b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/IdentifierExpression.java index a9d228ee..5f22df5b 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/IdentifierExpression.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/IdentifierExpression.java @@ -4,6 +4,7 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Value; +import lombok.experimental.NonFinal; import org.hypertrace.core.documentstore.expression.type.GroupTypeExpression; import org.hypertrace.core.documentstore.expression.type.SelectTypeExpression; import org.hypertrace.core.documentstore.expression.type.SortTypeExpression; @@ -17,7 +18,8 @@ *

Example: IdentifierExpression.of("col1"); */ @Value -@AllArgsConstructor(access = AccessLevel.PRIVATE) +@NonFinal +@AllArgsConstructor(access = AccessLevel.PROTECTED) public class IdentifierExpression implements GroupTypeExpression, SelectTypeExpression, SortTypeExpression { diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/SubQueryJoinExpression.java b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/SubQueryJoinExpression.java new file mode 100644 index 00000000..46a0de10 --- /dev/null +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/SubQueryJoinExpression.java @@ -0,0 +1,36 @@ +package org.hypertrace.core.documentstore.expression.impl; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Value; +import org.hypertrace.core.documentstore.expression.type.FilterTypeExpression; +import org.hypertrace.core.documentstore.expression.type.FromTypeExpression; +import org.hypertrace.core.documentstore.parser.FromTypeExpressionVisitor; +import org.hypertrace.core.documentstore.query.Query; + +/** + * Expression representing a join operation where the right side expression is a subquery. Note that + * this currently supports a self-join only, so the collection to be joined with is implicit. + * + *

For an example of using this expression, see the testSelfJoinWithSubQuery method in + * DocStoreQueryV1Test. + */ +@Value +@Builder(toBuilder = true) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +public class SubQueryJoinExpression implements FromTypeExpression { + Query subQuery; + String subQueryAlias; + FilterTypeExpression joinCondition; + + @Override + public T accept(FromTypeExpressionVisitor visitor) { + return visitor.visit(this); + } + + @Override + public String toString() { + return String.format("JOIN (%s) AS %s ON (%s)", subQuery, subQueryAlias, joinCondition); + } +} diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/MongoUtils.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/MongoUtils.java index 6efb08d9..f12868cf 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/MongoUtils.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/MongoUtils.java @@ -65,6 +65,13 @@ public static String decodeKey(final String key) { return key.replace("\\u002e", FIELD_SEPARATOR).replace("\\u0024", PREFIX).replace("\\\\", "\\"); } + public static String encodeVariableName(final String variableName) { + if (variableName == null) { + return null; + } + return variableName.replace(".", "_"); + } + public static String getLastField(final String fieldPath) { final String[] fields = fieldPath.split("\\" + FIELD_SEPARATOR); return fields[fields.length - 1]; diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/MongoAggregationPipelineConverter.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/MongoAggregationPipelineConverter.java new file mode 100644 index 00000000..dcad498f --- /dev/null +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/MongoAggregationPipelineConverter.java @@ -0,0 +1,147 @@ +package org.hypertrace.core.documentstore.mongo.query; + +import static java.util.Collections.singleton; +import static java.util.function.Function.identity; +import static java.util.function.Predicate.not; +import static java.util.stream.Collectors.toUnmodifiableList; +import static org.hypertrace.core.documentstore.mongo.query.MongoPaginationHelper.getLimitClause; +import static org.hypertrace.core.documentstore.mongo.query.MongoPaginationHelper.getSkipClause; +import static org.hypertrace.core.documentstore.mongo.query.parser.MongoFilterTypeExpressionParser.getFilterClause; +import static org.hypertrace.core.documentstore.mongo.query.parser.MongoNonProjectedSortTypeExpressionParser.getNonProjectedSortClause; +import static org.hypertrace.core.documentstore.mongo.query.parser.MongoSelectTypeExpressionParser.getProjectClause; +import static org.hypertrace.core.documentstore.mongo.query.parser.MongoSortTypeExpressionParser.getSortClause; + +import com.mongodb.BasicDBObject; +import com.mongodb.client.MongoCollection; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.hypertrace.core.documentstore.model.config.AggregatePipelineMode; +import org.hypertrace.core.documentstore.mongo.query.parser.AliasParser; +import org.hypertrace.core.documentstore.mongo.query.parser.MongoFromTypeExpressionParser; +import org.hypertrace.core.documentstore.mongo.query.parser.MongoGroupTypeExpressionParser; +import org.hypertrace.core.documentstore.parser.AggregateExpressionChecker; +import org.hypertrace.core.documentstore.parser.FunctionExpressionChecker; +import org.hypertrace.core.documentstore.query.Query; +import org.hypertrace.core.documentstore.query.SelectionSpec; +import org.hypertrace.core.documentstore.query.SortingSpec; + +@Slf4j +@AllArgsConstructor +public class MongoAggregationPipelineConverter { + private final AggregatePipelineMode aggregationPipelineMode; + private final MongoCollection collection; + + private final List>> + DEFAULT_AGGREGATE_PIPELINE_FUNCTIONS = + List.of( + query -> singleton(getFilterClause(query, Query::getFilter)), + query -> new MongoFromTypeExpressionParser(this).getFromClauses(query), + MongoGroupTypeExpressionParser::getGroupClauses, + query -> singleton(getProjectClause(query)), + query -> singleton(getFilterClause(query, Query::getAggregationFilter)), + query -> singleton(getSortClause(query)), + query -> singleton(getSkipClause(query)), + query -> singleton(getLimitClause(query))); + + private final List>> + SORT_OPTIMISED_AGGREGATE_PIPELINE_FUNCTIONS = + List.of( + query -> singleton(getFilterClause(query, Query::getFilter)), + query -> new MongoFromTypeExpressionParser(this).getFromClauses(query), + query -> singleton(getNonProjectedSortClause(query)), + query -> singleton(getSkipClause(query)), + query -> singleton(getLimitClause(query)), + query -> singleton(getProjectClause(query))); + + public String getCollectionName() { + return collection.getNamespace().getCollectionName(); + } + + public List convertToAggregatePipeline(Query query) { + List>> aggregatePipeline = + getAggregationPipeline(query); + + List pipeline = + aggregatePipeline.stream() + .flatMap(function -> function.apply(query).stream()) + .filter(not(BasicDBObject::isEmpty)) + .collect(toUnmodifiableList()); + return pipeline; + } + + private List>> getAggregationPipeline(Query query) { + List>> aggregatePipeline = + DEFAULT_AGGREGATE_PIPELINE_FUNCTIONS; + if (aggregationPipelineMode.equals(AggregatePipelineMode.SORT_OPTIMIZED_IF_POSSIBLE) + && query.getAggregations().isEmpty() + && query.getAggregationFilter().isEmpty() + && !isProjectionContainsAggregation(query) + && !isSortContainsAggregation(query)) { + log.debug("Using sort optimized aggregate pipeline functions for query: {}", query); + aggregatePipeline = SORT_OPTIMISED_AGGREGATE_PIPELINE_FUNCTIONS; + } + return aggregatePipeline; + } + + private boolean isProjectionContainsAggregation(Query query) { + return query.getSelections().stream() + .map(SelectionSpec::getExpression) + .anyMatch(spec -> spec.accept(new AggregateExpressionChecker())); + } + + private boolean isSortContainsAggregation(Query query) { + // ideally there should be only one alias per selection, + // in case of duplicates, we will accept the latest one + Map aliasToSelectionMap = + query.getSelections().stream() + .filter(spec -> this.getAlias(spec).isPresent()) + .collect( + Collectors.toMap( + entry -> this.getAlias(entry).orElseThrow(), identity(), (v1, v2) -> v2)); + return query.getSorts().stream() + .anyMatch(sort -> isSortOnAggregatedField(aliasToSelectionMap, sort)); + } + + private boolean isSortOnAggregatedField( + Map aliasToSelectionMap, SortingSpec sort) { + boolean isFunctionExpression = sort.getExpression().accept(new FunctionExpressionChecker()); + boolean isAggregateExpression = sort.getExpression().accept(new AggregateExpressionChecker()); + return isFunctionExpression + || isAggregateExpression + || isSortOnAggregatedProjection(aliasToSelectionMap, sort); + } + + private Optional getAlias(SelectionSpec selectionSpec) { + if (selectionSpec.getAlias() != null) { + return Optional.of(selectionSpec.getAlias()); + } + + return selectionSpec.getExpression().accept(new AliasParser()); + } + + private boolean isSortOnAggregatedProjection( + Map aliasToSelectionMap, SortingSpec sort) { + Optional alias = sort.getExpression().accept(new AliasParser()); + if (alias.isEmpty()) { + throw new UnsupportedOperationException( + "Cannot sort by an expression that does not have an alias in selection"); + } + + SelectionSpec selectionSpec = aliasToSelectionMap.get(alias.get()); + if (selectionSpec == null) { + return false; + } + + Boolean isFunctionExpression = + selectionSpec.getExpression().accept(new FunctionExpressionChecker()); + Boolean isAggregationExpression = + selectionSpec.getExpression().accept(new AggregateExpressionChecker()); + return isFunctionExpression || isAggregationExpression; + } +} diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/MongoQueryExecutor.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/MongoQueryExecutor.java index 40890779..60b15929 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/MongoQueryExecutor.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/MongoQueryExecutor.java @@ -1,26 +1,17 @@ package org.hypertrace.core.documentstore.mongo.query; import static java.lang.Long.parseLong; -import static java.util.Collections.singleton; import static java.util.Comparator.comparing; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.function.Function.identity; import static java.util.function.Predicate.not; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toUnmodifiableList; import static org.hypertrace.core.documentstore.model.options.QueryOptions.DEFAULT_QUERY_OPTIONS; import static org.hypertrace.core.documentstore.mongo.clause.MongoCountClauseSupplier.COUNT_ALIAS; import static org.hypertrace.core.documentstore.mongo.clause.MongoCountClauseSupplier.getCountClause; import static org.hypertrace.core.documentstore.mongo.query.MongoPaginationHelper.applyPagination; -import static org.hypertrace.core.documentstore.mongo.query.MongoPaginationHelper.getLimitClause; -import static org.hypertrace.core.documentstore.mongo.query.MongoPaginationHelper.getSkipClause; import static org.hypertrace.core.documentstore.mongo.query.parser.MongoFilterTypeExpressionParser.getFilter; -import static org.hypertrace.core.documentstore.mongo.query.parser.MongoFilterTypeExpressionParser.getFilterClause; -import static org.hypertrace.core.documentstore.mongo.query.parser.MongoNonProjectedSortTypeExpressionParser.getNonProjectedSortClause; -import static org.hypertrace.core.documentstore.mongo.query.parser.MongoSelectTypeExpressionParser.getProjectClause; import static org.hypertrace.core.documentstore.mongo.query.parser.MongoSelectTypeExpressionParser.getSelections; import static org.hypertrace.core.documentstore.mongo.query.parser.MongoSortTypeExpressionParser.getOrders; -import static org.hypertrace.core.documentstore.mongo.query.parser.MongoSortTypeExpressionParser.getSortClause; import com.mongodb.BasicDBObject; import com.mongodb.ServerAddress; @@ -29,58 +20,23 @@ import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCursor; import java.time.Duration; -import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.bson.conversions.Bson; -import org.hypertrace.core.documentstore.model.config.AggregatePipelineMode; import org.hypertrace.core.documentstore.model.config.ConnectionConfig; import org.hypertrace.core.documentstore.model.options.QueryOptions; import org.hypertrace.core.documentstore.mongo.collection.MongoCollectionOptionsApplier; -import org.hypertrace.core.documentstore.mongo.query.parser.AliasParser; -import org.hypertrace.core.documentstore.mongo.query.parser.MongoFromTypeExpressionParser; -import org.hypertrace.core.documentstore.mongo.query.parser.MongoGroupTypeExpressionParser; import org.hypertrace.core.documentstore.mongo.query.transformer.MongoQueryTransformer; -import org.hypertrace.core.documentstore.parser.AggregateExpressionChecker; -import org.hypertrace.core.documentstore.parser.FunctionExpressionChecker; import org.hypertrace.core.documentstore.query.Pagination; import org.hypertrace.core.documentstore.query.Query; -import org.hypertrace.core.documentstore.query.SelectionSpec; -import org.hypertrace.core.documentstore.query.SortingSpec; @Slf4j @AllArgsConstructor public class MongoQueryExecutor { - private static final List>> - DEFAULT_AGGREGATE_PIPELINE_FUNCTIONS = - List.of( - query -> singleton(getFilterClause(query, Query::getFilter)), - MongoFromTypeExpressionParser::getFromClauses, - MongoGroupTypeExpressionParser::getGroupClauses, - query -> singleton(getProjectClause(query)), - query -> singleton(getFilterClause(query, Query::getAggregationFilter)), - query -> singleton(getSortClause(query)), - query -> singleton(getSkipClause(query)), - query -> singleton(getLimitClause(query))); - - private static final List>> - SORT_OPTIMISED_AGGREGATE_PIPELINE_FUNCTIONS = - List.of( - query -> singleton(getFilterClause(query, Query::getFilter)), - MongoFromTypeExpressionParser::getFromClauses, - query -> singleton(getNonProjectedSortClause(query)), - query -> singleton(getSkipClause(query)), - query -> singleton(getLimitClause(query)), - query -> singleton(getProjectClause(query))); - private static final Integer ZERO = Integer.valueOf(0); private static final MongoCursor EMPTY_CURSOR = new MongoCursor<>() { @@ -125,6 +81,17 @@ public ServerAddress getServerAddress() { private final MongoCollectionOptionsApplier optionsApplier = new MongoCollectionOptionsApplier(); private final com.mongodb.client.MongoCollection collection; private final ConnectionConfig connectionConfig; + private final MongoAggregationPipelineConverter pipelineConverter; + + public MongoQueryExecutor( + com.mongodb.client.MongoCollection collection, + ConnectionConfig connectionConfig) { + this.collection = collection; + this.connectionConfig = connectionConfig; + this.pipelineConverter = + new MongoAggregationPipelineConverter( + connectionConfig.aggregationPipelineMode(), collection); + } public MongoCursor find(final Query query) { BasicDBObject filterClause = getFilter(query, Query::getFilter); @@ -157,14 +124,7 @@ public MongoCursor aggregate( Query query = transformAndLog(originalQuery); - List>> aggregatePipeline = - getAggregationPipeline(query); - - List pipeline = - aggregatePipeline.stream() - .flatMap(function -> function.apply(query).stream()) - .filter(not(BasicDBObject::isEmpty)) - .collect(toUnmodifiableList()); + List pipeline = pipelineConverter.convertToAggregatePipeline(query); logPipeline(pipeline, queryOptions); @@ -195,8 +155,7 @@ public long count(final Query originalQuery, final QueryOptions queryOptions) { final List pipeline = Stream.concat( - DEFAULT_AGGREGATE_PIPELINE_FUNCTIONS.stream() - .flatMap(function -> function.apply(query).stream()), + pipelineConverter.convertToAggregatePipeline(query).stream(), Stream.of(getCountClause())) .filter(not(BasicDBObject::isEmpty)) .collect(toList()); @@ -220,6 +179,10 @@ public long count(final Query originalQuery, final QueryOptions queryOptions) { return 0; } + public String getCollectionName() { + return collection.getNamespace().getCollectionName(); + } + private void logClauses( final Query query, final Bson projection, @@ -252,78 +215,6 @@ private Query transformAndLog(Query query) { return query; } - private List>> getAggregationPipeline(Query query) { - List>> aggregatePipeline = - DEFAULT_AGGREGATE_PIPELINE_FUNCTIONS; - if (connectionConfig - .aggregationPipelineMode() - .equals(AggregatePipelineMode.SORT_OPTIMIZED_IF_POSSIBLE) - && query.getAggregations().isEmpty() - && query.getAggregationFilter().isEmpty() - && !isProjectionContainsAggregation(query) - && !isSortContainsAggregation(query)) { - log.debug("Using sort optimized aggregate pipeline functions for query: {}", query); - aggregatePipeline = SORT_OPTIMISED_AGGREGATE_PIPELINE_FUNCTIONS; - } - return aggregatePipeline; - } - - private boolean isProjectionContainsAggregation(Query query) { - return query.getSelections().stream() - .map(SelectionSpec::getExpression) - .anyMatch(spec -> spec.accept(new AggregateExpressionChecker())); - } - - private boolean isSortContainsAggregation(Query query) { - // ideally there should be only one alias per selection, - // in case of duplicates, we will accept the latest one - Map aliasToSelectionMap = - query.getSelections().stream() - .filter(spec -> this.getAlias(spec).isPresent()) - .collect( - Collectors.toMap( - entry -> this.getAlias(entry).orElseThrow(), identity(), (v1, v2) -> v2)); - return query.getSorts().stream() - .anyMatch(sort -> isSortOnAggregatedField(aliasToSelectionMap, sort)); - } - - private boolean isSortOnAggregatedField( - Map aliasToSelectionMap, SortingSpec sort) { - boolean isFunctionExpression = sort.getExpression().accept(new FunctionExpressionChecker()); - boolean isAggregateExpression = sort.getExpression().accept(new AggregateExpressionChecker()); - return isFunctionExpression - || isAggregateExpression - || isSortOnAggregatedProjection(aliasToSelectionMap, sort); - } - - private Optional getAlias(SelectionSpec selectionSpec) { - if (selectionSpec.getAlias() != null) { - return Optional.of(selectionSpec.getAlias()); - } - - return selectionSpec.getExpression().accept(new AliasParser()); - } - - private boolean isSortOnAggregatedProjection( - Map aliasToSelectionMap, SortingSpec sort) { - Optional alias = sort.getExpression().accept(new AliasParser()); - if (alias.isEmpty()) { - throw new UnsupportedOperationException( - "Cannot sort by an expression that does not have an alias in selection"); - } - - SelectionSpec selectionSpec = aliasToSelectionMap.get(alias.get()); - if (selectionSpec == null) { - return false; - } - - Boolean isFunctionExpression = - selectionSpec.getExpression().accept(new FunctionExpressionChecker()); - Boolean isAggregationExpression = - selectionSpec.getExpression().accept(new AggregateExpressionChecker()); - return isFunctionExpression || isAggregationExpression; - } - private Duration queryTimeout( final ConnectionConfig connectionConfig, final QueryOptions queryOptions) { return Stream.of(connectionConfig.queryTimeout(), queryOptions.queryTimeout()) diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/AliasParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/AliasParser.java index 80e29d6e..550f6ac1 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/AliasParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/AliasParser.java @@ -2,12 +2,14 @@ import java.util.Optional; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; import org.hypertrace.core.documentstore.parser.SelectTypeExpressionVisitor; import org.hypertrace.core.documentstore.parser.SortTypeExpressionVisitor; +import org.hypertrace.core.documentstore.query.SelectionSpec; public class AliasParser implements SelectTypeExpressionVisitor, SortTypeExpressionVisitor { @@ -40,4 +42,10 @@ public Optional visit(ConstantExpression expression) { public Optional visit(DocumentConstantExpression expression) { return Optional.empty(); } + + @SuppressWarnings("unchecked") + @Override + public Optional visit(final AliasedIdentifierExpression expression) { + throw new UnsupportedOperationException("This operation is not supported"); + } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoDollarPrefixingIdempotentParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoDollarPrefixingIdempotentParser.java index b2dbbbdb..8bfa6330 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoDollarPrefixingIdempotentParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoDollarPrefixingIdempotentParser.java @@ -3,6 +3,7 @@ import static org.hypertrace.core.documentstore.mongo.MongoUtils.PREFIX; import java.util.Optional; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; final class MongoDollarPrefixingIdempotentParser extends MongoSelectTypeExpressionParser { @@ -19,6 +20,15 @@ public String visit(final IdentifierExpression expression) { .orElse(null); } + @SuppressWarnings("unchecked") + @Override + public String visit(final AliasedIdentifierExpression expression) { + return Optional.ofNullable(baseParser.visit(expression)) + .map(Object::toString) + .map(identifier -> PREFIX + identifier) + .orElse(null); + } + private String idempotentPrefix(final String identifier) { return identifier.startsWith(PREFIX) ? identifier : PREFIX + identifier; } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoEmptySelectionTypeParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoEmptySelectionTypeParser.java new file mode 100644 index 00000000..b5061610 --- /dev/null +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoEmptySelectionTypeParser.java @@ -0,0 +1,42 @@ +package org.hypertrace.core.documentstore.mongo.query.parser; + +import java.util.Collections; +import java.util.Map; +import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; +import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; +import org.hypertrace.core.documentstore.parser.SelectTypeExpressionVisitor; + +class MongoEmptySelectionTypeParser implements SelectTypeExpressionVisitor { + @Override + public Map visit(AggregateExpression expression) { + return Collections.emptyMap(); + } + + @Override + public Map visit(ConstantExpression expression) { + return Collections.emptyMap(); + } + + @Override + public Map visit(ConstantExpression.DocumentConstantExpression expression) { + return Collections.emptyMap(); + } + + @Override + public Map visit(FunctionExpression expression) { + return Collections.emptyMap(); + } + + @Override + public Map visit(IdentifierExpression expression) { + return Collections.emptyMap(); + } + + @Override + public Map visit(AliasedIdentifierExpression expression) { + return Collections.emptyMap(); + } +} diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoFromTypeExpressionParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoFromTypeExpressionParser.java index 96a0f5ce..7d7e407d 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoFromTypeExpressionParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoFromTypeExpressionParser.java @@ -2,10 +2,19 @@ import com.mongodb.BasicDBObject; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.hypertrace.core.documentstore.expression.impl.SubQueryJoinExpression; import org.hypertrace.core.documentstore.expression.impl.UnnestExpression; +import org.hypertrace.core.documentstore.expression.type.FilterTypeExpression; +import org.hypertrace.core.documentstore.mongo.MongoUtils; +import org.hypertrace.core.documentstore.mongo.query.MongoAggregationPipelineConverter; +import org.hypertrace.core.documentstore.mongo.query.parser.filter.MongoRelationalFilterParserFactory.FilterLocation; +import org.hypertrace.core.documentstore.mongo.query.parser.filter.MongoRelationalFilterParserFactory.MongoRelationalFilterContext; +import org.hypertrace.core.documentstore.mongo.query.transformer.MongoQueryTransformer; +import org.hypertrace.core.documentstore.parser.FilterTypeExpressionVisitor; import org.hypertrace.core.documentstore.parser.FromTypeExpressionVisitor; import org.hypertrace.core.documentstore.query.Query; @@ -14,10 +23,27 @@ public class MongoFromTypeExpressionParser implements FromTypeExpressionVisitor private static final String PATH_KEY = "path"; public static final String PRESERVE_NULL_AND_EMPTY_ARRAYS = "preserveNullAndEmptyArrays"; private static final String UNWIND_OPERATOR = "$unwind"; + private static final String LOOKUP_OPERATOR = "$lookup"; + private static final String MATCH_OPERATOR = "$match"; + private static final String REPLACE_ROOT_OPERATOR = "$replaceRoot"; + private static final String EXPR_OPERATOR = "$expr"; + private static final String LOOKUP_FROM_FIELD = "from"; + private static final String LOOKUP_LET_FIELD = "let"; + private static final String LOOKUP_PIPELINE_FIELD = "pipeline"; + private static final String LOOKUP_AS_FIELD = "as"; + private static final String NEW_ROOT_FIELD = "newRoot"; + private static final String JOINED_RESULT_FIELD_NAME_PREFIX = "__joined_result_with_"; private static final MongoIdentifierPrefixingParser mongoIdentifierPrefixingParser = new MongoIdentifierPrefixingParser(new MongoIdentifierExpressionParser()); + private final MongoAggregationPipelineConverter pipelineConverter; + private MongoLetClauseBuilder mongoLetClauseBuilder; + + public MongoFromTypeExpressionParser(MongoAggregationPipelineConverter pipelineConverter) { + this.pipelineConverter = pipelineConverter; + } + @SuppressWarnings("unchecked") @Override public List visit(UnnestExpression unnestExpression) { @@ -42,9 +68,104 @@ public List visit(UnnestExpression unnestExpression) { return objects; } - public static List getFromClauses(final Query query) { + @SuppressWarnings("unchecked") + @Override + public List visit(SubQueryJoinExpression subQueryJoinExpression) { + this.mongoLetClauseBuilder = + new MongoLetClauseBuilder(subQueryJoinExpression.getSubQueryAlias()); + + Query transformedSubQuery = + MongoQueryTransformer.transform(subQueryJoinExpression.getSubQuery()); + List aggregatePipeline = + new ArrayList<>(pipelineConverter.convertToAggregatePipeline(transformedSubQuery)); + + // This is the field name in which the joined results will be put after the lookup stage. This + // field name can be used by subsequent stages in the aggregate pipeline to do operations on the + // joined result. + String joinedResultFieldName = + getJoinedResultFieldName(subQueryJoinExpression.getSubQueryAlias()); + // Add the lookup stage to join the subquery results with the main collection + aggregatePipeline.add(createLookupStage(subQueryJoinExpression, joinedResultFieldName)); + + // Lookup Stage puts the joined results into an array field. We need to unwind that array field + // to get the joined results as separate documents. + aggregatePipeline.add(createUnwindStage(joinedResultFieldName)); + + // Replace root with the joined document + aggregatePipeline.add(createReplaceRootStage(joinedResultFieldName)); + + return Collections.unmodifiableList(aggregatePipeline); + } + + private BasicDBObject createLookupStage( + SubQueryJoinExpression subQueryJoinExpression, String joinedResultFieldName) { + BasicDBObject lookupStage = new BasicDBObject(); + BasicDBObject lookupSpec = new BasicDBObject(); + + lookupSpec.put(LOOKUP_FROM_FIELD, pipelineConverter.getCollectionName()); + lookupSpec.put( + LOOKUP_LET_FIELD, subQueryJoinExpression.getJoinCondition().accept(mongoLetClauseBuilder)); + lookupSpec.put(LOOKUP_PIPELINE_FIELD, createLookupPipeline(subQueryJoinExpression)); + lookupSpec.put(LOOKUP_AS_FIELD, joinedResultFieldName); + + lookupStage.put(LOOKUP_OPERATOR, lookupSpec); + return lookupStage; + } + + private List createLookupPipeline(SubQueryJoinExpression subQueryJoinExpression) { + return List.of(createMatchStage(subQueryJoinExpression)); + } + + private BasicDBObject createMatchStage(SubQueryJoinExpression subQueryJoinExpression) { + BasicDBObject matchStage = new BasicDBObject(); + BasicDBObject expr = new BasicDBObject(); + expr.put(EXPR_OPERATOR, getFilterClause(subQueryJoinExpression.getJoinCondition())); + matchStage.put(MATCH_OPERATOR, expr); + return matchStage; + } + + private BasicDBObject getFilterClause(FilterTypeExpression joinCondition) { + final FilterTypeExpressionVisitor parser = + new MongoFilterTypeExpressionParser( + MongoRelationalFilterContext.builder() + .location(FilterLocation.INSIDE_EXPR) + .lhsParser( + new MongoDollarPrefixingIdempotentParser(new MongoIdentifierExpressionParser())) + .rhsParser( + new MongoDollarPrefixingIdempotentParser(new MongoIdentifierExpressionParser())) + .build()); + final Map filter = joinCondition.accept(parser); + return new BasicDBObject(filter); + } + + private String getJoinedResultFieldName(String subQueryAlias) { + return JOINED_RESULT_FIELD_NAME_PREFIX + subQueryAlias; + } + + private BasicDBObject createUnwindStage(String joinedResultFieldName) { + BasicDBObject unwindStage = new BasicDBObject(); + BasicDBObject unwindSpec = new BasicDBObject(); + + unwindSpec.put(PATH_KEY, MongoUtils.PREFIX + joinedResultFieldName); + unwindSpec.put(PRESERVE_NULL_AND_EMPTY_ARRAYS, true); + + unwindStage.put(UNWIND_OPERATOR, unwindSpec); + return unwindStage; + } + + private BasicDBObject createReplaceRootStage(String joinedResultFieldName) { + BasicDBObject replaceRootStage = new BasicDBObject(); + BasicDBObject newRoot = new BasicDBObject(); + + newRoot.put(NEW_ROOT_FIELD, MongoUtils.PREFIX + joinedResultFieldName); + replaceRootStage.put(REPLACE_ROOT_OPERATOR, newRoot); + + return replaceRootStage; + } + + public List getFromClauses(final Query query) { MongoFromTypeExpressionParser mongoFromTypeExpressionParser = - new MongoFromTypeExpressionParser(); + new MongoFromTypeExpressionParser(pipelineConverter); return query.getFromTypeExpressions().stream() .flatMap( fromTypeExpression -> diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoIdentifierExpressionParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoIdentifierExpressionParser.java index 40d1323f..80e09302 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoIdentifierExpressionParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoIdentifierExpressionParser.java @@ -1,7 +1,9 @@ package org.hypertrace.core.documentstore.mongo.query.parser; import lombok.NoArgsConstructor; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; +import org.hypertrace.core.documentstore.mongo.MongoUtils; @NoArgsConstructor public final class MongoIdentifierExpressionParser extends MongoSelectTypeExpressionParser { @@ -16,7 +18,17 @@ public String visit(final IdentifierExpression expression) { return parse(expression); } + @SuppressWarnings("unchecked") + @Override + public String visit(final AliasedIdentifierExpression expression) { + return MongoUtils.PREFIX + MongoUtils.encodeVariableName(parse(expression)); + } + String parse(final IdentifierExpression expression) { return expression.getName(); } + + String parse(final AliasedIdentifierExpression expression) { + return expression.getName(); + } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoLetClauseBuilder.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoLetClauseBuilder.java new file mode 100644 index 00000000..fd84f1fa --- /dev/null +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoLetClauseBuilder.java @@ -0,0 +1,70 @@ +package org.hypertrace.core.documentstore.mongo.query.parser; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ArrayRelationalFilterExpression; +import org.hypertrace.core.documentstore.expression.impl.DocumentArrayFilterExpression; +import org.hypertrace.core.documentstore.expression.impl.KeyExpression; +import org.hypertrace.core.documentstore.expression.impl.LogicalExpression; +import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; +import org.hypertrace.core.documentstore.expression.type.FilterTypeExpression; +import org.hypertrace.core.documentstore.mongo.MongoUtils; +import org.hypertrace.core.documentstore.parser.FilterTypeExpressionVisitor; + +// Visitor for building the let clause in lookup stage from filter expressions +class MongoLetClauseBuilder implements FilterTypeExpressionVisitor { + + private final String subQueryAlias; + + MongoLetClauseBuilder(String subQueryAlias) { + this.subQueryAlias = subQueryAlias; + } + + @Override + public Map visit(LogicalExpression expression) { + Map letClause = new HashMap<>(); + for (FilterTypeExpression operand : expression.getOperands()) { + letClause.putAll(operand.accept(this)); + } + return Collections.unmodifiableMap(letClause); + } + + @Override + public Map visit(RelationalExpression expression) { + Map letClause = new HashMap<>(); + letClause.putAll(expression.getLhs().accept(new MongoLetClauseSelectTypeExpressionVisitor())); + letClause.putAll(expression.getRhs().accept(new MongoLetClauseSelectTypeExpressionVisitor())); + return Collections.unmodifiableMap(letClause); + } + + @Override + public Map visit(KeyExpression expression) { + return Collections.emptyMap(); + } + + @Override + public Map visit(ArrayRelationalFilterExpression expression) { + return Collections.emptyMap(); + } + + @Override + public Map visit(DocumentArrayFilterExpression expression) { + return Collections.emptyMap(); + } + + private class MongoLetClauseSelectTypeExpressionVisitor extends MongoEmptySelectionTypeParser { + + @Override + public Map visit(AliasedIdentifierExpression aliasedExpression) { + Map letClause = new HashMap<>(); + if (aliasedExpression.getContextAlias().equals(subQueryAlias)) { + letClause.put( + MongoUtils.encodeVariableName(aliasedExpression.getName()), + MongoUtils.PREFIX + aliasedExpression.getName()); + } + return Collections.unmodifiableMap(letClause); + } + } +} diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoNonProjectedSortTypeExpressionParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoNonProjectedSortTypeExpressionParser.java index cf5c4710..bd9b8949 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoNonProjectedSortTypeExpressionParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoNonProjectedSortTypeExpressionParser.java @@ -13,6 +13,7 @@ import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; @@ -93,6 +94,12 @@ public Map visit(DocumentConstantExpression expression) { expression.getValue().toString())); } + @SuppressWarnings("unchecked") + @Override + public Map visit(final AliasedIdentifierExpression expression) { + throw new UnsupportedOperationException("This operation is not supported"); + } + public static BasicDBObject getNonProjectedSortClause(final Query query) { BasicDBObject orders = getOrders(query); return orders.isEmpty() ? orders : new BasicDBObject(SORT_CLAUSE, orders); diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoSelectTypeExpressionParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoSelectTypeExpressionParser.java index 3a561fc0..2cf00262 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoSelectTypeExpressionParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoSelectTypeExpressionParser.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; @@ -59,6 +60,11 @@ public T visit(final IdentifierExpression expression) { return baseParser.visit(expression); } + @Override + public T visit(final AliasedIdentifierExpression expression) { + return baseParser.visit(expression); + } + public static BasicDBObject getSelections(final Query query) { List selectionSpecs = query.getSelections(); MongoSelectTypeExpressionParser parser = diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/transformer/MongoSelectionsAddingTransformation.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/transformer/MongoSelectionsAddingTransformation.java index 0a876eb2..2fb4ca0c 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/transformer/MongoSelectionsAddingTransformation.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/transformer/MongoSelectionsAddingTransformation.java @@ -8,6 +8,7 @@ import java.util.Optional; import lombok.AllArgsConstructor; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; @@ -123,4 +124,10 @@ public Optional visit(final FunctionExpression expression) { public Optional visit(final IdentifierExpression expression) { return Optional.empty(); } + + @SuppressWarnings("unchecked") + @Override + public Optional visit(final AliasedIdentifierExpression expression) { + throw new UnsupportedOperationException("This operation is not supported"); + } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/transformer/MongoSelectionsUpdatingTransformation.java b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/transformer/MongoSelectionsUpdatingTransformation.java index b099542c..ea38a3a1 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/transformer/MongoSelectionsUpdatingTransformation.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/transformer/MongoSelectionsUpdatingTransformation.java @@ -15,6 +15,7 @@ import java.util.Set; import java.util.function.Function; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; @@ -132,6 +133,12 @@ public SelectionSpec visit(final IdentifierExpression expression) { return SelectionSpec.of(IdentifierExpression.of(identifier), alias); } + @SuppressWarnings("unchecked") + @Override + public SelectionSpec visit(final AliasedIdentifierExpression expression) { + throw new UnsupportedOperationException("This operation is not supported"); + } + private SelectionSpec substitute(final AggregateExpression expression) { return Optional.ofNullable(AGGREGATION_SUBSTITUTE_MAP.get(expression.getAggregator())) .map(converter -> SelectionSpec.of(converter.apply(expression), source.getAlias())) diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/parser/AggregateExpressionChecker.java b/document-store/src/main/java/org/hypertrace/core/documentstore/parser/AggregateExpressionChecker.java index b8c0ae2f..8b70235f 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/parser/AggregateExpressionChecker.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/parser/AggregateExpressionChecker.java @@ -1,6 +1,7 @@ package org.hypertrace.core.documentstore.parser; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; @@ -34,4 +35,9 @@ public Boolean visit(FunctionExpression expression) { public Boolean visit(IdentifierExpression expression) { return false; } + + @Override + public Boolean visit(AliasedIdentifierExpression expression) { + throw new UnsupportedOperationException("This operation is not supported"); + } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/parser/FromTypeExpressionVisitor.java b/document-store/src/main/java/org/hypertrace/core/documentstore/parser/FromTypeExpressionVisitor.java index ecb84b6b..f5bb4be6 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/parser/FromTypeExpressionVisitor.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/parser/FromTypeExpressionVisitor.java @@ -1,7 +1,10 @@ package org.hypertrace.core.documentstore.parser; +import org.hypertrace.core.documentstore.expression.impl.SubQueryJoinExpression; import org.hypertrace.core.documentstore.expression.impl.UnnestExpression; public interface FromTypeExpressionVisitor { T visit(UnnestExpression unnestExpression); + + T visit(SubQueryJoinExpression subQueryJoinExpression); } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/parser/FunctionExpressionChecker.java b/document-store/src/main/java/org/hypertrace/core/documentstore/parser/FunctionExpressionChecker.java index 15c19baa..e9779879 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/parser/FunctionExpressionChecker.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/parser/FunctionExpressionChecker.java @@ -1,6 +1,7 @@ package org.hypertrace.core.documentstore.parser; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; @@ -34,4 +35,9 @@ public Boolean visit(FunctionExpression expression) { public Boolean visit(IdentifierExpression expression) { return false; } + + @Override + public Boolean visit(AliasedIdentifierExpression expression) { + throw new UnsupportedOperationException("This operation is not supported"); + } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/parser/SelectTypeExpressionVisitor.java b/document-store/src/main/java/org/hypertrace/core/documentstore/parser/SelectTypeExpressionVisitor.java index 45890c73..a6e0ed0e 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/parser/SelectTypeExpressionVisitor.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/parser/SelectTypeExpressionVisitor.java @@ -1,6 +1,7 @@ package org.hypertrace.core.documentstore.parser; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; @@ -16,4 +17,6 @@ public interface SelectTypeExpressionVisitor { T visit(final FunctionExpression expression); T visit(final IdentifierExpression expression); + + T visit(final AliasedIdentifierExpression expression); } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java index 457e577f..7052b90a 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresSelectionQueryTransformer.java @@ -5,6 +5,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; @@ -109,6 +110,11 @@ public Boolean visit(FunctionExpression expression) { public Boolean visit(IdentifierExpression expression) { return false; } + + @Override + public Boolean visit(AliasedIdentifierExpression expression) { + throw new UnsupportedOperationException("This operation is not supported"); + } } private static class LocalSelectTypeIdentifierExpressionSelector @@ -137,6 +143,11 @@ public Boolean visit(FunctionExpression expression) { public Boolean visit(IdentifierExpression expression) { return true; } + + @Override + public Boolean visit(AliasedIdentifierExpression expression) { + throw new UnsupportedOperationException("This operation is not supported"); + } } private static class LocalGroupByIdentifierExpressionSelector diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresUnnestQueryTransformer.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresUnnestQueryTransformer.java index fd4c2f64..8145b8f7 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresUnnestQueryTransformer.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/transformer/PostgresUnnestQueryTransformer.java @@ -9,6 +9,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ArrayRelationalFilterExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; @@ -18,6 +19,7 @@ import org.hypertrace.core.documentstore.expression.impl.KeyExpression; import org.hypertrace.core.documentstore.expression.impl.LogicalExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; +import org.hypertrace.core.documentstore.expression.impl.SubQueryJoinExpression; import org.hypertrace.core.documentstore.expression.impl.UnnestExpression; import org.hypertrace.core.documentstore.expression.operators.LogicalOperator; import org.hypertrace.core.documentstore.expression.type.FilterTypeExpression; @@ -179,6 +181,11 @@ private static class UnnestExpressionProvider implements FromTypeExpressionVisit public UnnestExpression visit(UnnestExpression unnestExpression) { return unnestExpression; } + + @Override + public String visit(SubQueryJoinExpression subQueryJoinExpression) { + throw new UnsupportedOperationException("This operation is not supported"); + } } @SuppressWarnings("unchecked") @@ -211,6 +218,11 @@ public List visit(FunctionExpression expression) { public List visit(IdentifierExpression expression) { return List.of(expression.getName()); } + + @Override + public List visit(AliasedIdentifierExpression expression) { + throw new UnsupportedOperationException("This operation is not supported"); + } } @SuppressWarnings("unchecked") diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFromTypeExpressionVisitor.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFromTypeExpressionVisitor.java index 40dad767..db5a09c6 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFromTypeExpressionVisitor.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresFromTypeExpressionVisitor.java @@ -4,6 +4,7 @@ import java.util.stream.Collectors; import lombok.Getter; import org.apache.commons.lang3.StringUtils; +import org.hypertrace.core.documentstore.expression.impl.SubQueryJoinExpression; import org.hypertrace.core.documentstore.expression.impl.UnnestExpression; import org.hypertrace.core.documentstore.parser.FromTypeExpressionVisitor; import org.hypertrace.core.documentstore.postgres.query.v1.PostgresQueryParser; @@ -62,6 +63,11 @@ public String visit(UnnestExpression unnestExpression) { return String.format(fmt, newTable, preTable, tableAlias, unwindExpr, unwindExprAlias); } + @Override + public String visit(SubQueryJoinExpression subQueryJoinExpression) { + throw new UnsupportedOperationException("This operation is not supported"); + } + public static Optional getFromClause(PostgresQueryParser postgresQueryParser) { PostgresFromTypeExpressionVisitor postgresFromTypeExpressionVisitor = diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresSelectTypeExpressionVisitor.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresSelectTypeExpressionVisitor.java index ae1cf863..44b7f454 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresSelectTypeExpressionVisitor.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresSelectTypeExpressionVisitor.java @@ -6,6 +6,7 @@ import lombok.AllArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; import org.hypertrace.core.documentstore.expression.impl.ConstantExpression.DocumentConstantExpression; import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; @@ -64,6 +65,11 @@ public T visit(final IdentifierExpression expression) { return baseVisitor.visit(expression); } + @Override + public T visit(final AliasedIdentifierExpression expression) { + throw new UnsupportedOperationException("This operation is not supported"); + } + public abstract PostgresQueryParser getPostgresQueryParser(); @AllArgsConstructor diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresUnnestFilterTypeExpressionVisitor.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresUnnestFilterTypeExpressionVisitor.java index fcc6404a..34724321 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresUnnestFilterTypeExpressionVisitor.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/vistors/PostgresUnnestFilterTypeExpressionVisitor.java @@ -3,6 +3,7 @@ import java.util.Optional; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.hypertrace.core.documentstore.expression.impl.SubQueryJoinExpression; import org.hypertrace.core.documentstore.expression.impl.UnnestExpression; import org.hypertrace.core.documentstore.parser.FromTypeExpressionVisitor; import org.hypertrace.core.documentstore.postgres.query.v1.PostgresQueryParser; @@ -23,6 +24,11 @@ public String visit(UnnestExpression unnestExpression) { return where.orElse(""); } + @Override + public String visit(SubQueryJoinExpression subQueryJoinExpression) { + throw new UnsupportedOperationException("This operation is not supported"); + } + public static Optional getFilterClause(PostgresQueryParser postgresQueryParser) { PostgresUnnestFilterTypeExpressionVisitor postgresUnnestFilterTypeExpressionVisitor = new PostgresUnnestFilterTypeExpressionVisitor(postgresQueryParser);