Skip to content

Commit a25ae95

Browse files
normen662normen662
committed
basic aggregation queries using agg indexes intersections work
1 parent 6cadd5a commit a25ae95

File tree

13 files changed

+164
-58
lines changed

13 files changed

+164
-58
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Compensation.java

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -641,13 +641,15 @@ default Compensation intersect(@Nonnull Compensation otherCompensation) {
641641
final var combinedPredicateMap = new LinkedIdentityMap<QueryPredicate, PredicateCompensationFunction>();
642642
for (final var entry : getPredicateCompensationMap().entrySet()) {
643643
// if the other side does not have compensation for this key, we don't need compensation
644-
if (otherCompensationMap.containsKey(entry.getKey())) {
644+
final var otherPredicateCompensationFunction = otherCompensationMap.get(entry.getKey());
645+
if (otherPredicateCompensationFunction != null) {
645646
// Both compensations have a compensation for this particular predicate which is essentially
646-
// reapplying the predicate. At this point it doesn't matter which side we take as both create
647-
// the same compensating filter. If at any point in the future one data access has a better
648-
// reapplication we need to generate plan variants with either compensation and let the planner
649-
// figure out which one wins.
650-
// We just pick one side here.
647+
// reapplying the predicate. Three cases arise:
648+
// 1. Both predicate compensation functions are needed and possible. At this point it doesn't
649+
// matter which side we take as both create the same compensating filter. If at any point in the
650+
// future one data access has a better reapplication we need to generate plan variants with
651+
// either compensation and let the planner figure out which one wins. We just pick one side here.
652+
// 2. TODO.
651653
combinedPredicateMap.put(entry.getKey(),
652654
entry.getValue().amend(unmatchedAggregateMap, newMatchedAggregateMap));
653655
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PredicateMultiMap.java

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -164,9 +164,14 @@ PredicateCompensationFunction amend(@Nonnull BiMap<CorrelationIdentifier, Value>
164164
@Nonnull
165165
Set<QueryPredicate> applyCompensationForPredicate(@Nonnull TranslationMap translationMap);
166166

167+
@Nonnull
168+
static PredicateCompensationFunction ofPredicate(@Nonnull final QueryPredicate predicate) {
169+
return ofPredicate(predicate, false);
170+
}
171+
167172
@Nonnull
168173
static PredicateCompensationFunction ofPredicate(@Nonnull final QueryPredicate predicate,
169-
@Nonnull final BiFunction<QueryPredicate, TranslationMap, Set<QueryPredicate>> compensationFunction) {
174+
final boolean shouldSimplifyValues) {
170175
final var isImpossible = predicateContainsUnmatchedValues(predicate);
171176

172177
return new PredicateCompensationFunction() {
@@ -189,13 +194,13 @@ public PredicateCompensationFunction amend(@Nonnull final BiMap<CorrelationIdent
189194
Optional.of(amendValue(unmatchedAggregateMap, amendedMatchedAggregateMap,
190195
rootValue)));
191196
Verify.verify(amendedTranslatedPredicateOptional.isPresent());
192-
return ofPredicate(amendedTranslatedPredicateOptional.get(), compensationFunction);
197+
return ofPredicate(amendedTranslatedPredicateOptional.get(), true);
193198
}
194199

195200
@Nonnull
196201
@Override
197202
public Set<QueryPredicate> applyCompensationForPredicate(@Nonnull final TranslationMap translationMap) {
198-
return compensationFunction.apply(predicate, translationMap);
203+
return LinkedIdentitySet.of(predicate.translateCorrelations(translationMap, shouldSimplifyValues));
199204
}
200205
};
201206
}
@@ -369,8 +374,12 @@ ResultCompensationFunction amend(@Nonnull BiMap<CorrelationIdentifier, Value> un
369374
Value applyCompensationForResult(@Nonnull TranslationMap translationMap);
370375

371376
@Nonnull
372-
static ResultCompensationFunction ofValue(@Nonnull final Value value,
373-
@Nonnull final BiFunction<Value, TranslationMap, Value> compensationFunction) {
377+
static ResultCompensationFunction ofValue(@Nonnull final Value value) {
378+
return ofValue(value, false);
379+
}
380+
381+
@Nonnull
382+
static ResultCompensationFunction ofValue(@Nonnull final Value value, final boolean shouldSimplifyValue) {
374383
final var isImpossible = valueContainsUnmatchedValues(value);
375384

376385
return new ResultCompensationFunction() {
@@ -390,14 +399,13 @@ public ResultCompensationFunction amend(@Nonnull final BiMap<CorrelationIdentifi
390399
@Nonnull final Map<Value, Value> amendedMatchedAggregateMap) {
391400
final var amendedTranslatedQueryValue =
392401
amendValue(unmatchedAggregateMap, amendedMatchedAggregateMap, value);
393-
394-
return ofValue(amendedTranslatedQueryValue, compensationFunction);
402+
return ofValue(amendedTranslatedQueryValue, true);
395403
}
396404

397405
@Nonnull
398406
@Override
399407
public Value applyCompensationForResult(@Nonnull final TranslationMap translationMap) {
400-
return compensationFunction.apply(value, translationMap);
408+
return value.translateCorrelations(translationMap, shouldSimplifyValue);
401409
}
402410
};
403411
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/GroupByExpression.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -614,9 +614,7 @@ public Compensation compensate(@Nonnull final PartialMatch partialMatch,
614614
final var pulledUpTranslatedResultValue = pulledUpTranslatedResultValueOptional.get();
615615

616616
resultCompensationFunction =
617-
ResultCompensationFunction.ofValue(pulledUpTranslatedResultValue,
618-
(value, translationMap) -> value.translateCorrelations(translationMap,
619-
false));
617+
ResultCompensationFunction.ofValue(pulledUpTranslatedResultValue);
620618
isCompensationImpossible |= resultCompensationFunction.isImpossible();
621619

622620
pulledUpAggregateMappings =

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/LogicalTypeFilterExpression.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -219,10 +219,7 @@ public Compensation compensate(@Nonnull final PartialMatch partialMatch,
219219

220220
final var pulledUpTranslatedResultValue = pulledUpTranslatedResultValueOptional.get();
221221

222-
resultCompensationFunction =
223-
ResultCompensationFunction.ofValue(pulledUpTranslatedResultValue,
224-
(value, translationMap) -> value.translateCorrelations(translationMap,
225-
false));
222+
resultCompensationFunction = ResultCompensationFunction.ofValue(pulledUpTranslatedResultValue);
226223
isCompensationImpossible |= resultCompensationFunction.isImpossible();
227224

228225
aggregateMappings =

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -842,21 +842,21 @@ public Compensation compensate(@Nonnull final PartialMatch partialMatch,
842842
break;
843843
}
844844

845+
if (compensationFunction == null) {
846+
compensationFunction = compensationFunctionForCandidatePredicate;
847+
}
848+
845849
if (!compensationFunctionForCandidatePredicate.isImpossible()) {
846850
isCompensationFunctionImpossible = false;
847-
if (compensationFunction == null) {
848-
compensationFunction = compensationFunctionForCandidatePredicate;
849-
}
850851
}
851852
}
852853

853854
if (isCompensationFunctionNeeded) {
854855
isAnyCompensationFunctionNeeded = true;
855856
if (isCompensationFunctionImpossible) {
856857
isAnyCompensationFunctionImpossible = true;
857-
} else {
858-
predicateCompensationMap.put(predicate, Objects.requireNonNull(compensationFunction));
859858
}
859+
predicateCompensationMap.put(predicate, Objects.requireNonNull(compensationFunction));
860860
}
861861
}
862862
}
@@ -877,10 +877,7 @@ public Compensation compensate(@Nonnull final PartialMatch partialMatch,
877877

878878
final var pulledUpTranslatedResultValue = pulledUpTranslatedResultValueOptional.get();
879879

880-
resultCompensationFunction =
881-
ResultCompensationFunction.ofValue(pulledUpTranslatedResultValue,
882-
(value, translationMap) -> value.translateCorrelations(translationMap,
883-
false));
880+
resultCompensationFunction = ResultCompensationFunction.ofValue(pulledUpTranslatedResultValue);
884881
isAnyCompensationFunctionImpossible |= resultCompensationFunction.isImpossible();
885882

886883
aggregateMappings =

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/LeafQueryPredicate.java

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import com.apple.foundationdb.annotation.API;
2424
import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange;
2525
import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
26-
import com.apple.foundationdb.record.query.plan.cascades.LinkedIdentitySet;
2726
import com.apple.foundationdb.record.query.plan.cascades.PartialMatch;
2827
import com.apple.foundationdb.record.query.plan.cascades.PredicateMultiMap;
2928
import com.apple.foundationdb.record.query.plan.cascades.values.translation.PullUp;
@@ -82,11 +81,7 @@ default PredicateMultiMap.PredicateCompensationFunction computeCompensationFunct
8281
default PredicateMultiMap.PredicateCompensationFunction computeCompensationFunctionForLeaf(@Nonnull final PullUp pullUp) {
8382
return toResidualPredicate()
8483
.replaceValuesMaybe(pullUp::pullUpMaybe)
85-
.map(queryPredicate ->
86-
PredicateMultiMap.PredicateCompensationFunction.ofPredicate(queryPredicate,
87-
(pulledUpPredicate, translationMap) ->
88-
LinkedIdentitySet.of(pulledUpPredicate.translateCorrelations(translationMap,
89-
false))))
84+
.map(PredicateMultiMap.PredicateCompensationFunction::ofPredicate)
9085
.orElse(PredicateMultiMap.PredicateCompensationFunction.impossibleCompensation());
9186
}
9287
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/QueryPredicate.java

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,6 @@
3434
import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange;
3535
import com.apple.foundationdb.record.query.plan.cascades.Correlated;
3636
import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
37-
import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence;
38-
import com.apple.foundationdb.record.query.plan.cascades.LinkedIdentitySet;
3937
import com.apple.foundationdb.record.query.plan.cascades.Narrowable;
4038
import com.apple.foundationdb.record.query.plan.cascades.PartialMatch;
4139
import com.apple.foundationdb.record.query.plan.cascades.PredicateMultiMap.PredicateCompensation;
@@ -48,6 +46,7 @@
4846
import com.apple.foundationdb.record.query.plan.cascades.values.Value;
4947
import com.apple.foundationdb.record.query.plan.cascades.values.translation.PullUp;
5048
import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap;
49+
import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence;
5150
import com.apple.foundationdb.record.query.plan.serialization.PlanSerialization;
5251
import com.google.common.base.Verify;
5352
import com.google.common.collect.ImmutableList;
@@ -250,11 +249,7 @@ default PredicateCompensationFunction computeCompensationFunction(@Nonnull final
250249

251250
return toResidualPredicate()
252251
.replaceValuesMaybe(pullUp::pullUpMaybe)
253-
.map(queryPredicate ->
254-
PredicateCompensationFunction.ofPredicate(queryPredicate,
255-
(pulledUpPredicate, translationMap) ->
256-
LinkedIdentitySet.of(pulledUpPredicate.translateCorrelations(translationMap,
257-
false))))
252+
.map(PredicateCompensationFunction::ofPredicate)
258253
.orElse(PredicateCompensationFunction.impossibleCompensation());
259254
}
260255

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import com.apple.foundationdb.record.query.plan.TextScan;
2929
import com.apple.foundationdb.record.query.plan.bitmap.ComposedBitmapIndexQueryPlan;
3030
import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
31+
import com.apple.foundationdb.record.query.plan.cascades.Quantifier;
3132
import com.apple.foundationdb.record.query.plan.plans.RecordQueryAggregateIndexPlan;
3233
import com.apple.foundationdb.record.query.plan.plans.RecordQueryComparatorPlan;
3334
import com.apple.foundationdb.record.query.plan.plans.RecordQueryCoveringIndexPlan;
@@ -81,6 +82,7 @@
8182

8283
import javax.annotation.Nonnull;
8384
import java.util.Arrays;
85+
import java.util.List;
8486
import java.util.function.Supplier;
8587

8688
/**
@@ -331,8 +333,26 @@ public ExplainTokens visitInComparandJoinPlan(@Nonnull final RecordQueryInCompar
331333

332334
@Nonnull
333335
@Override
334-
public ExplainTokens visitMultiIntersectionOnValuesPlan(@Nonnull final RecordQueryMultiIntersectionOnValuesPlan element) {
335-
return new ExplainTokens().addToString("TODO"); // TODO
336+
public ExplainTokens visitMultiIntersectionOnValuesPlan(@Nonnull final RecordQueryMultiIntersectionOnValuesPlan multiIntersectionOnValuesPlan) {
337+
visitAndJoin(() -> new ExplainTokens().addWhitespace().addToString("∩").addWhitespace(),
338+
multiIntersectionOnValuesPlan.getChildren());
339+
final var compareByExplainTokens = new ExplainTokens().addWhitespace().addKeyword("COMPARE")
340+
.addWhitespace().addKeyword("BY").addWhitespace()
341+
.addNested(multiIntersectionOnValuesPlan.getComparisonKeyFunction().explain().getExplainTokens());
342+
addNested(ExplainLevel.SOME_DETAILS, compareByExplainTokens);
343+
344+
final List<? extends Quantifier> quantifiers = multiIntersectionOnValuesPlan.getQuantifiers();
345+
final var withExplainTokens =
346+
new ExplainTokens().addWhitespace().addKeyword("WITH").addWhitespace()
347+
.addSequence(() -> new ExplainTokens().addCommaAndWhiteSpace(),
348+
() -> quantifiers.stream()
349+
.map(quantifier -> new ExplainTokens()
350+
.addAliasDefinition(quantifier.getAlias()))
351+
.iterator());
352+
addNested(ExplainLevel.SOME_DETAILS, withExplainTokens);
353+
final var returnExplainTokens = new ExplainTokens().addWhitespace().addKeyword("RETURN")
354+
.addWhitespace().addNested(multiIntersectionOnValuesPlan.getResultValue().explain().getExplainTokens());
355+
return addNested(ExplainLevel.SOME_DETAILS, returnExplainTokens);
336356
}
337357

338358
@Nonnull
@@ -435,10 +455,10 @@ public ExplainTokens visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan
435455
private ExplainTokens visitIntersectionPlan(@Nonnull final RecordQueryIntersectionPlan intersectionPlan) {
436456
visitAndJoin(() -> new ExplainTokens().addWhitespace().addToString("∩").addWhitespace(),
437457
intersectionPlan.getChildren());
438-
final var comparyByExplainTokens = new ExplainTokens().addWhitespace().addKeyword("COMPARE")
458+
final var compareByExplainTokens = new ExplainTokens().addWhitespace().addKeyword("COMPARE")
439459
.addWhitespace().addKeyword("BY").addWhitespace()
440460
.addNested(intersectionPlan.getComparisonKeyFunction().explain().getExplainTokens());
441-
return addNested(ExplainLevel.SOME_DETAILS, comparyByExplainTokens);
461+
return addNested(ExplainLevel.SOME_DETAILS, compareByExplainTokens);
442462
}
443463

444464
@Nonnull
@@ -576,10 +596,10 @@ private ExplainTokens visitUnionPlan(@Nonnull final RecordQueryUnionPlan unionPl
576596
visitAndJoin(() -> new ExplainTokens().addWhitespace().addToString("∪").addWhitespace(),
577597
unionPlan.getChildren());
578598

579-
final var comparyByExplainTokens = new ExplainTokens().addWhitespace().addKeyword("COMPARE")
599+
final var compareByExplainTokens = new ExplainTokens().addWhitespace().addKeyword("COMPARE")
580600
.addWhitespace().addKeyword("BY").addWhitespace()
581601
.addNested(unionPlan.getComparisonKeyFunction().explain().getExplainTokens());
582-
return addNested(ExplainLevel.SOME_DETAILS, comparyByExplainTokens);
602+
return addNested(ExplainLevel.SOME_DETAILS, compareByExplainTokens);
583603
}
584604

585605
@Nonnull

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,6 @@ public Set<Type> getDynamicTypes() {
166166
return collectionValue.getDynamicTypes();
167167
}
168168

169-
170169
@Nonnull
171170
@Override
172171
public String toString() {

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryMultiIntersectionOnValuesPlan.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import com.google.common.collect.ImmutableList;
4949
import com.google.common.collect.ImmutableMap;
5050
import com.google.common.collect.ImmutableSet;
51+
import com.google.common.collect.Streams;
5152
import com.google.protobuf.Message;
5253

5354
import javax.annotation.Nonnull;
@@ -57,6 +58,7 @@
5758
import java.util.Set;
5859
import java.util.function.Function;
5960
import java.util.stream.Collectors;
61+
import java.util.stream.Stream;
6062

6163
/**
6264
* Intersection plan that compares using a {@link Value}.
@@ -128,7 +130,7 @@ public List<? extends Value> getComparisonKeyValues() {
128130
@Nonnull
129131
@Override
130132
public Set<Type> getDynamicTypes() {
131-
return getComparisonKeyValues().stream()
133+
return Streams.concat(getComparisonKeyValues().stream(), Stream.of(resultValue))
132134
.flatMap(comparisonKeyValue ->
133135
comparisonKeyValue.getDynamicTypes().stream()).collect(ImmutableSet.toImmutableSet());
134136
}

0 commit comments

Comments
 (0)