Skip to content

Commit 9458645

Browse files
normen662normen662
committed
basic aggregation queries using agg indexes intersections work
1 parent 59ae202 commit 9458645

File tree

8 files changed

+60
-38
lines changed

8 files changed

+60
-38
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Compensation.java

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -646,13 +646,15 @@ default Compensation intersect(@Nonnull Compensation otherCompensation) {
646646
final var combinedPredicateMap = new LinkedIdentityMap<QueryPredicate, PredicateCompensationFunction>();
647647
for (final var entry : getPredicateCompensationMap().entrySet()) {
648648
// if the other side does not have compensation for this key, we don't need compensation
649-
if (otherCompensationMap.containsKey(entry.getKey())) {
649+
final var otherPredicateCompensationFunction = otherCompensationMap.get(entry.getKey());
650+
if (otherPredicateCompensationFunction != null) {
650651
// Both compensations have a compensation for this particular predicate which is essentially
651-
// reapplying the predicate. At this point it doesn't matter which side we take as both create
652-
// the same compensating filter. If at any point in the future one data access has a better
653-
// reapplication we need to generate plan variants with either compensation and let the planner
654-
// figure out which one wins.
655-
// We just pick one side here.
652+
// reapplying the predicate. Three cases arise:
653+
// 1. Both predicate compensation functions are needed and possible. At this point it doesn't
654+
// matter which side we take as both create the same compensating filter. If at any point in the
655+
// future one data access has a better reapplication we need to generate plan variants with
656+
// either compensation and let the planner figure out which one wins. We just pick one side here.
657+
// 2. TODO.
656658
combinedPredicateMap.put(entry.getKey(),
657659
entry.getValue().amend(unmatchedAggregateMap, newMatchedAggregateMap));
658660
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PredicateMultiMap.java

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -164,9 +164,14 @@ PredicateCompensationFunction amend(@Nonnull BiMap<CorrelationIdentifier, Value>
164164
@Nonnull
165165
Set<QueryPredicate> applyCompensationForPredicate(@Nonnull TranslationMap translationMap);
166166

167+
@Nonnull
168+
static PredicateCompensationFunction ofPredicate(@Nonnull final QueryPredicate predicate) {
169+
return ofPredicate(predicate, false);
170+
}
171+
167172
@Nonnull
168173
static PredicateCompensationFunction ofPredicate(@Nonnull final QueryPredicate predicate,
169-
@Nonnull final BiFunction<QueryPredicate, TranslationMap, Set<QueryPredicate>> compensationFunction) {
174+
final boolean shouldSimplifyValues) {
170175
final var isImpossible = predicateContainsUnmatchedValues(predicate);
171176

172177
return new PredicateCompensationFunction() {
@@ -189,13 +194,13 @@ public PredicateCompensationFunction amend(@Nonnull final BiMap<CorrelationIdent
189194
Optional.of(amendValue(unmatchedAggregateMap, amendedMatchedAggregateMap,
190195
rootValue)));
191196
Verify.verify(amendedTranslatedPredicateOptional.isPresent());
192-
return ofPredicate(amendedTranslatedPredicateOptional.get(), compensationFunction);
197+
return ofPredicate(amendedTranslatedPredicateOptional.get(), true);
193198
}
194199

195200
@Nonnull
196201
@Override
197202
public Set<QueryPredicate> applyCompensationForPredicate(@Nonnull final TranslationMap translationMap) {
198-
return compensationFunction.apply(predicate, translationMap);
203+
return LinkedIdentitySet.of(predicate.translateCorrelations(translationMap, shouldSimplifyValues));
199204
}
200205
};
201206
}
@@ -376,8 +381,12 @@ static ResultCompensationFunction ofTranslation(@Nonnull final Value resultValue
376381
}
377382

378383
@Nonnull
379-
static ResultCompensationFunction ofValue(@Nonnull final Value value,
380-
@Nonnull final BiFunction<Value, TranslationMap, Value> compensationFunction) {
384+
static ResultCompensationFunction ofValue(@Nonnull final Value value) {
385+
return ofValue(value, false);
386+
}
387+
388+
@Nonnull
389+
static ResultCompensationFunction ofValue(@Nonnull final Value value, final boolean shouldSimplifyValue) {
381390
final var isImpossible = valueContainsUnmatchedValues(value);
382391

383392
return new ResultCompensationFunction() {
@@ -397,14 +406,13 @@ public ResultCompensationFunction amend(@Nonnull final BiMap<CorrelationIdentifi
397406
@Nonnull final Map<Value, Value> amendedMatchedAggregateMap) {
398407
final var amendedTranslatedQueryValue =
399408
amendValue(unmatchedAggregateMap, amendedMatchedAggregateMap, value);
400-
401-
return ofValue(amendedTranslatedQueryValue, compensationFunction);
409+
return ofValue(amendedTranslatedQueryValue, true);
402410
}
403411

404412
@Nonnull
405413
@Override
406414
public Value applyCompensationForResult(@Nonnull final TranslationMap translationMap) {
407-
return compensationFunction.apply(value, translationMap);
415+
return value.translateCorrelations(translationMap, shouldSimplifyValue);
408416
}
409417
};
410418
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/ExplainPlanVisitor.java

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import com.apple.foundationdb.record.query.plan.explain.ExplainSymbolMap;
3737
import com.apple.foundationdb.record.query.plan.explain.ExplainTokens;
3838
import com.apple.foundationdb.record.query.plan.explain.PrettyExplainFormatter;
39+
import com.apple.foundationdb.record.query.plan.cascades.Quantifier;
3940
import com.apple.foundationdb.record.query.plan.plans.RecordQueryAggregateIndexPlan;
4041
import com.apple.foundationdb.record.query.plan.plans.RecordQueryComparatorPlan;
4142
import com.apple.foundationdb.record.query.plan.plans.RecordQueryCoveringIndexPlan;
@@ -90,6 +91,7 @@
9091

9192
import javax.annotation.Nonnull;
9293
import java.util.Arrays;
94+
import java.util.List;
9395
import java.util.function.Supplier;
9496

9597
/**
@@ -341,8 +343,26 @@ public ExplainTokens visitInComparandJoinPlan(@Nonnull final RecordQueryInCompar
341343

342344
@Nonnull
343345
@Override
344-
public ExplainTokens visitMultiIntersectionOnValuesPlan(@Nonnull final RecordQueryMultiIntersectionOnValuesPlan element) {
345-
return new ExplainTokens().addToString("TODO"); // TODO
346+
public ExplainTokens visitMultiIntersectionOnValuesPlan(@Nonnull final RecordQueryMultiIntersectionOnValuesPlan multiIntersectionOnValuesPlan) {
347+
visitAndJoin(() -> new ExplainTokens().addWhitespace().addToString("∩").addWhitespace(),
348+
multiIntersectionOnValuesPlan.getChildren());
349+
final var compareByExplainTokens = new ExplainTokens().addWhitespace().addKeyword("COMPARE")
350+
.addWhitespace().addKeyword("BY").addWhitespace()
351+
.addNested(multiIntersectionOnValuesPlan.getComparisonKeyFunction().explain().getExplainTokens());
352+
addNested(ExplainLevel.SOME_DETAILS, compareByExplainTokens);
353+
354+
final List<? extends Quantifier> quantifiers = multiIntersectionOnValuesPlan.getQuantifiers();
355+
final var withExplainTokens =
356+
new ExplainTokens().addWhitespace().addKeyword("WITH").addWhitespace()
357+
.addSequence(() -> new ExplainTokens().addCommaAndWhiteSpace(),
358+
() -> quantifiers.stream()
359+
.map(quantifier -> new ExplainTokens()
360+
.addAliasDefinition(quantifier.getAlias()))
361+
.iterator());
362+
addNested(ExplainLevel.SOME_DETAILS, withExplainTokens);
363+
final var returnExplainTokens = new ExplainTokens().addWhitespace().addKeyword("RETURN")
364+
.addWhitespace().addNested(multiIntersectionOnValuesPlan.getResultValue().explain().getExplainTokens());
365+
return addNested(ExplainLevel.SOME_DETAILS, returnExplainTokens);
346366
}
347367

348368
@Nonnull
@@ -452,10 +472,10 @@ public ExplainTokens visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan
452472
private ExplainTokens visitIntersectionPlan(@Nonnull final RecordQueryIntersectionPlan intersectionPlan) {
453473
visitAndJoin(() -> new ExplainTokens().addWhitespace().addToString("∩").addWhitespace(),
454474
intersectionPlan.getChildren());
455-
final var comparyByExplainTokens = new ExplainTokens().addWhitespace().addKeyword("COMPARE")
475+
final var compareByExplainTokens = new ExplainTokens().addWhitespace().addKeyword("COMPARE")
456476
.addWhitespace().addKeyword("BY").addWhitespace()
457477
.addNested(intersectionPlan.getComparisonKeyFunction().explain().getExplainTokens());
458-
return addNested(ExplainLevel.SOME_DETAILS, comparyByExplainTokens);
478+
return addNested(ExplainLevel.SOME_DETAILS, compareByExplainTokens);
459479
}
460480

461481
@Nonnull
@@ -593,10 +613,10 @@ private ExplainTokens visitUnionPlan(@Nonnull final RecordQueryUnionPlan unionPl
593613
visitAndJoin(() -> new ExplainTokens().addWhitespace().addToString("∪").addWhitespace(),
594614
unionPlan.getChildren());
595615

596-
final var comparyByExplainTokens = new ExplainTokens().addWhitespace().addKeyword("COMPARE")
616+
final var compareByExplainTokens = new ExplainTokens().addWhitespace().addKeyword("COMPARE")
597617
.addWhitespace().addKeyword("BY").addWhitespace()
598618
.addNested(unionPlan.getComparisonKeyFunction().explain().getExplainTokens());
599-
return addNested(ExplainLevel.SOME_DETAILS, comparyByExplainTokens);
619+
return addNested(ExplainLevel.SOME_DETAILS, compareByExplainTokens);
600620
}
601621

602622
@Nonnull

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -844,21 +844,21 @@ public Compensation compensate(@Nonnull final PartialMatch partialMatch,
844844
break;
845845
}
846846

847+
if (compensationFunction == null) {
848+
compensationFunction = compensationFunctionForCandidatePredicate;
849+
}
850+
847851
if (!compensationFunctionForCandidatePredicate.isImpossible()) {
848852
isCompensationFunctionImpossible = false;
849-
if (compensationFunction == null) {
850-
compensationFunction = compensationFunctionForCandidatePredicate;
851-
}
852853
}
853854
}
854855

855856
if (isCompensationFunctionNeeded) {
856857
isAnyCompensationFunctionNeeded = true;
857858
if (isCompensationFunctionImpossible) {
858859
isAnyCompensationFunctionImpossible = true;
859-
} else {
860-
predicateCompensationMap.put(predicate, Objects.requireNonNull(compensationFunction));
861860
}
861+
predicateCompensationMap.put(predicate, Objects.requireNonNull(compensationFunction));
862862
}
863863
}
864864
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/LeafQueryPredicate.java

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import com.apple.foundationdb.annotation.API;
2424
import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange;
2525
import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
26-
import com.apple.foundationdb.record.query.plan.cascades.LinkedIdentitySet;
2726
import com.apple.foundationdb.record.query.plan.cascades.PartialMatch;
2827
import com.apple.foundationdb.record.query.plan.cascades.PredicateMultiMap;
2928
import com.apple.foundationdb.record.query.plan.cascades.values.translation.PullUp;
@@ -82,11 +81,7 @@ default PredicateMultiMap.PredicateCompensationFunction computeCompensationFunct
8281
default PredicateMultiMap.PredicateCompensationFunction computeCompensationFunctionForLeaf(@Nonnull final PullUp pullUp) {
8382
return toResidualPredicate()
8483
.replaceValuesMaybe(pullUp::pullUpMaybe)
85-
.map(queryPredicate ->
86-
PredicateMultiMap.PredicateCompensationFunction.ofPredicate(queryPredicate,
87-
(pulledUpPredicate, translationMap) ->
88-
LinkedIdentitySet.of(pulledUpPredicate.translateCorrelations(translationMap,
89-
false))))
84+
.map(PredicateMultiMap.PredicateCompensationFunction::ofPredicate)
9085
.orElse(PredicateMultiMap.PredicateCompensationFunction.impossibleCompensation());
9186
}
9287
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/QueryPredicate.java

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -249,11 +249,7 @@ default PredicateCompensationFunction computeCompensationFunction(@Nonnull final
249249

250250
return toResidualPredicate()
251251
.replaceValuesMaybe(pullUp::pullUpMaybe)
252-
.map(queryPredicate ->
253-
PredicateCompensationFunction.ofPredicate(queryPredicate,
254-
(pulledUpPredicate, translationMap) ->
255-
LinkedIdentitySet.of(pulledUpPredicate.translateCorrelations(translationMap,
256-
false))))
252+
.map(PredicateCompensationFunction::ofPredicate)
257253
.orElse(PredicateCompensationFunction.impossibleCompensation());
258254
}
259255

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,6 @@ public Set<Type> getDynamicTypes() {
169169
return collectionValue.getDynamicTypes();
170170
}
171171

172-
173172
@Nonnull
174173
@Override
175174
public String toString() {

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryMultiIntersectionOnValuesPlan.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import com.google.common.collect.ImmutableList;
4949
import com.google.common.collect.ImmutableMap;
5050
import com.google.common.collect.ImmutableSet;
51+
import com.google.common.collect.Streams;
5152
import com.google.protobuf.Message;
5253

5354
import javax.annotation.Nonnull;
@@ -57,6 +58,7 @@
5758
import java.util.Set;
5859
import java.util.function.Function;
5960
import java.util.stream.Collectors;
61+
import java.util.stream.Stream;
6062

6163
/**
6264
* Intersection plan that compares using a {@link Value}.
@@ -128,7 +130,7 @@ public List<? extends Value> getComparisonKeyValues() {
128130
@Nonnull
129131
@Override
130132
public Set<Type> getDynamicTypes() {
131-
return getComparisonKeyValues().stream()
133+
return Streams.concat(getComparisonKeyValues().stream(), Stream.of(resultValue))
132134
.flatMap(comparisonKeyValue ->
133135
comparisonKeyValue.getDynamicTypes().stream()).collect(ImmutableSet.toImmutableSet());
134136
}

0 commit comments

Comments
 (0)