Skip to content

Commit d950cd0

Browse files
normen662normen662
committed
agg testcases run
1 parent 2f5e9b8 commit d950cd0

File tree

9 files changed

+195
-123
lines changed

9 files changed

+195
-123
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexMatchCandidate.java

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -206,8 +206,8 @@ private int getPermutedCount() {
206206
public List<MatchedOrderingPart> computeMatchedOrderingParts(@Nonnull final MatchInfo matchInfo,
207207
@Nonnull final List<CorrelationIdentifier> sortParameterIds,
208208
final boolean isReverse) {
209-
final var parameterBindingMap =
210-
matchInfo.getRegularMatchInfo().getParameterBindingMap();
209+
final var regularMatchInfo = matchInfo.getRegularMatchInfo();
210+
final var parameterBindingMap = regularMatchInfo.getParameterBindingMap();
211211
final var normalizedKeyExpressions =
212212
getFullKeyExpression().normalizeKeyForPositions();
213213

@@ -219,13 +219,17 @@ public List<MatchedOrderingPart> computeMatchedOrderingParts(@Nonnull final Matc
219219
final var deconstructedValues =
220220
Values.deconstructRecord(QuantifiedObjectValue.of(Quantifier.current(), selectHavingResultType));
221221

222+
final var rollUpToGroupingValues = regularMatchInfo.getRollUpToGroupingValues();
223+
final int orderSize = rollUpToGroupingValues != null ? rollUpToGroupingValues.size() : sortParameterIds.size();
224+
222225
// Compute the ordering for this index by collecting the result values of the selectHaving statement
223226
// associated with each sortParameterId. Note that for most aggregate indexes, the aggregate value is
224227
// in the FDB value, and so it does not contribute to the ordering of the index, so there is no sortParameterId
225228
// corresponding to it. For the PERMUTED_MIN and PERMUTED_MAX indexes, the aggregate _does_ have a corresponding
226229
// sortParameterId. Its position is determined by the permutedSize option, handled below by adjusting the
227230
// sortParameterId's index before looking it up in the original key expression
228-
for (final var parameterId : sortParameterIds) {
231+
for (int i = 0; i < orderSize; i++) {
232+
final var parameterId = sortParameterIds.get(i);
229233
final var ordinalInCandidate = candidateParameterIds.indexOf(parameterId);
230234
Verify.verify(ordinalInCandidate >= 0);
231235
int permutedIndex = indexWithPermutation(ordinalInCandidate);
@@ -425,6 +429,7 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch
425429
selectHavingResultValue,
426430
groupByResultValue,
427431
constraintMaybe);
432+
428433
if (regularMatchInfo.getRollUpToGroupingValues() != null) {
429434
//
430435
// We need to perform a roll up.
@@ -434,14 +439,13 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch
434439

435440
//final var recordValues = Values.deconstructRecord(recordValue);
436441
final var groupingAndAggregateAccessors =
437-
getGroupingAndAggregateAccessors(regularMatchInfo.getRollUpToGroupingValues().size(),
438-
aggregateIndexScanAlias);
442+
getGroupingAndAggregateAccessors(getGroupingCount(), aggregateIndexScanAlias);
439443
final var groupingAccessorValues = groupingAndAggregateAccessors.getLeft();
440444
final var aggregateAccessorValue = groupingAndAggregateAccessors.getRight();
441445
final var allFields = resultType.getFields();
442446
final var rollUpGroupingColumnsBuilder =
443447
ImmutableList.<Column<? extends Value>>builder();
444-
for (int i = 0; i < groupingAccessorValues.size(); i++) {
448+
for (int i = 0; i < regularMatchInfo.getRollUpToGroupingValues().size(); i++) {
445449
final var field = allFields.get(i);
446450
rollUpGroupingColumnsBuilder.add(Column.of(field, groupingAccessorValues.get(i)));
447451
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Compensation.java

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -634,6 +634,8 @@ default Compensation intersect(@Nonnull Compensation otherCompensation) {
634634
final var newGroupByMappings = GroupByMappings.of(newMatchedGroupingsMapBuilder.build(),
635635
newMatchedAggregatesMap, newUnmatchedAggregatesMapBuilder.build());
636636

637+
boolean isImpossible = false;
638+
637639
final ResultCompensationFunction newResultResultCompensationFunction;
638640
final var resultCompensationFunction = getResultCompensationFunction();
639641
final var otherResultCompensationFunction = otherWithSelectCompensation.getResultCompensationFunction();
@@ -646,6 +648,7 @@ default Compensation intersect(@Nonnull Compensation otherCompensation) {
646648
// pick the one from this side -- it does not matter as both candidates have the same shape
647649
newResultResultCompensationFunction =
648650
resultCompensationFunction.amend(unmatchedAggregateMap, newMatchedAggregatesMap);
651+
isImpossible |= newResultResultCompensationFunction.isImpossible();
649652
}
650653

651654
final var otherCompensationMap =
@@ -656,14 +659,17 @@ default Compensation intersect(@Nonnull Compensation otherCompensation) {
656659
final var otherPredicateCompensationFunction = otherCompensationMap.get(entry.getKey());
657660
if (otherPredicateCompensationFunction != null) {
658661
// Both compensations have a compensation for this particular predicate which is essentially
659-
// reapplying the predicate. Three cases arise:
662+
// reapplying the predicate. Two cases arise:
660663
// 1. Both predicate compensation functions are needed and possible. At this point it doesn't
661664
// matter which side we take as both create the same compensating filter. If at any point in the
662665
// future one data access has a better reapplication we need to generate plan variants with
663666
// either compensation and let the planner figure out which one wins. We just pick one side here.
664-
// 2. TODO.
665-
combinedPredicateMap.put(entry.getKey(),
666-
entry.getValue().amend(unmatchedAggregateMap, newMatchedAggregatesMap));
667+
// 2. Either one or both compensation functions are impossible, but thee intersection is possible.
668+
// We take the compensation function from this side and amend it with the compensation function
669+
// from the other side.
670+
final var newPredicateCompensationFunction = entry.getValue().amend(unmatchedAggregateMap, newMatchedAggregatesMap);
671+
combinedPredicateMap.put(entry.getKey(), newPredicateCompensationFunction);
672+
isImpossible |= newPredicateCompensationFunction.isImpossible();
667673
}
668674
}
669675

@@ -686,10 +692,12 @@ default Compensation intersect(@Nonnull Compensation otherCompensation) {
686692

687693
final var unmatchedQuantifiers = Sets.intersection(this.getUnmatchedQuantifiers(), otherWithSelectCompensation.getUnmatchedQuantifiers());
688694
final var unmatchedQuantifierAliases = unmatchedQuantifiers.stream().map(Quantifier::getAlias).collect(ImmutableList.toImmutableList());
689-
final var isImpossible = combinedPredicateMap.keySet()
690-
.stream()
691-
.flatMap(queryPredicate -> queryPredicate.getCorrelatedTo().stream())
692-
.anyMatch(unmatchedQuantifierAliases::contains);
695+
if (!isImpossible) {
696+
isImpossible = combinedPredicateMap.keySet()
697+
.stream()
698+
.flatMap(queryPredicate -> queryPredicate.getCorrelatedTo().stream())
699+
.anyMatch(unmatchedQuantifierAliases::contains);
700+
}
693701

694702
return intersectedChildCompensation.derived(isImpossible,
695703
combinedPredicateMap,

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -488,8 +488,10 @@ public Ordering visitInUnionOnKeyExpressionPlan(@Nonnull final RecordQueryInUnio
488488

489489
@Nonnull
490490
@Override
491-
public Ordering visitMultiIntersectionOnValuesPlan(@Nonnull final RecordQueryMultiIntersectionOnValuesPlan element) {
492-
return Ordering.empty(); // TODO
491+
public Ordering visitMultiIntersectionOnValuesPlan(@Nonnull final RecordQueryMultiIntersectionOnValuesPlan multiIntersectionOnValuesPlan) {
492+
final var orderings = orderingsFromChildren(multiIntersectionOnValuesPlan);
493+
return deriveForDistinctSetOperationFromOrderings(orderings,
494+
multiIntersectionOnValuesPlan.getComparisonKeyOrderingParts(), Ordering.INTERSECTION);
493495
}
494496

495497
@Nonnull

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java

Lines changed: 33 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,7 @@ protected Set<? extends RelationalExpression> dataAccessForMatchPartition(@Nonnu
338338
bestMatchToDistinctPlanMap,
339339
binaryPartition,
340340
requestedOrderings);
341-
if (binaryIntersections.hasCommonIntersectionOrdering()) {
341+
if (binaryIntersections.hasViableIntersection()) {
342342
updateIntersectionInfoMap(intersectionInfoMap, binaryPartition, binaryIntersections);
343343
} else {
344344
if (sieveBitMatrix != null) {
@@ -396,7 +396,10 @@ protected Set<? extends RelationalExpression> dataAccessForMatchPartition(@Nonnu
396396
kPartition,
397397
requestedOrderings);
398398

399-
Verify.verify(intersectionResult.hasCommonIntersectionOrdering());
399+
if (!intersectionResult.hasViableIntersection()) {
400+
continue;
401+
}
402+
400403
hasCommonOrderingForK = true;
401404

402405
updateIntersectionInfoMap(intersectionInfoMap, kPartition, intersectionResult);
@@ -433,14 +436,6 @@ protected static Optional<List<Value>> commonRecordKeyValuesMaybe(@Nonnull Itera
433436
}
434437
key = keyMaybe.get();
435438
} else if (matchCandidate instanceof AggregateIndexMatchCandidate) {
436-
// final var aggregateIndexMatchCandidate = (AggregateIndexMatchCandidate)matchCandidate;
437-
// final var rollUpToGroupingValues = regularMatchInfo.getRollUpToGroupingValues();
438-
// if (rollUpToGroupingValues == null) {
439-
// key = aggregateIndexMatchCandidate.getGroupByValues();
440-
// } else {
441-
// key = aggregateIndexMatchCandidate.getGroupByValues().subList(0, rollUpToGroupingValues.size());
442-
// }
443-
444439
final var aggregateIndexMatchCandidate = (AggregateIndexMatchCandidate)matchCandidate;
445440
final var rollUpToGroupingValues = regularMatchInfo.getRollUpToGroupingValues();
446441
if (rollUpToGroupingValues == null) {
@@ -583,8 +578,13 @@ private static List<SingleMatchedAccess> prepareMatchesAndCompensations(final @N
583578
final @Nonnull Set<RequestedOrdering> requestedOrderings) {
584579
final var partialMatchesWithCompensation = new ArrayList<SingleMatchedAccess>();
585580
for (final var partialMatch: partialMatches) {
581+
final var topToTopTranslationMapOptional = computeTopToTopTranslationMapMaybe(partialMatch);
582+
if (topToTopTranslationMapOptional.isEmpty()) {
583+
continue;
584+
}
585+
final var topToTopTranslationMap = topToTopTranslationMapOptional.get();
586586
final var satisfyingOrderingsPairOptional =
587-
satisfiesAnyRequestedOrderings(partialMatch, requestedOrderings);
587+
satisfiesAnyRequestedOrderings(partialMatch, topToTopTranslationMap, requestedOrderings);
588588
if (satisfyingOrderingsPairOptional.isEmpty()) {
589589
continue;
590590
}
@@ -604,7 +604,7 @@ private static List<SingleMatchedAccess> prepareMatchesAndCompensations(final @N
604604

605605
if (scanDirection == ScanDirection.FORWARD || scanDirection == ScanDirection.BOTH) {
606606
partialMatchesWithCompensation.add(new SingleMatchedAccess(partialMatch, compensation,
607-
topAlias, false, satisfyingOrderingsPair.getRight()));
607+
topAlias, false, topToTopTranslationMap, satisfyingOrderingsPair.getRight()));
608608
}
609609

610610
//
@@ -617,7 +617,7 @@ private static List<SingleMatchedAccess> prepareMatchesAndCompensations(final @N
617617
//
618618
if (scanDirection == ScanDirection.REVERSE /* || scanDirection == ScanDirection.BOTH */) {
619619
partialMatchesWithCompensation.add(new SingleMatchedAccess(partialMatch, compensation,
620-
topAlias, true, satisfyingOrderingsPair.getRight()));
620+
topAlias, true, topToTopTranslationMap, satisfyingOrderingsPair.getRight()));
621621
}
622622
}
623623

@@ -627,6 +627,12 @@ private static List<SingleMatchedAccess> prepareMatchesAndCompensations(final @N
627627
return partialMatchesWithCompensation;
628628
}
629629

630+
@Nonnull
631+
private static Optional<TranslationMap> computeTopToTopTranslationMapMaybe(final PartialMatch partialMatch) {
632+
final var maxMatchMap = partialMatch.getMatchInfo().getMaxMatchMap();
633+
return maxMatchMap.pullUpMaybe(Quantifier.current(), Quantifier.current());
634+
}
635+
630636
/**
631637
* Private helper method to compute the subset of orderings passed in that would be satisfied by a scan
632638
* if the given {@link PartialMatch} were to be planned.
@@ -639,21 +645,14 @@ private static List<SingleMatchedAccess> prepareMatchesAndCompensations(final @N
639645
@Nonnull
640646
@SuppressWarnings("java:S135")
641647
private static Optional<NonnullPair<ScanDirection, Set<RequestedOrdering>>> satisfiesAnyRequestedOrderings(@Nonnull final PartialMatch partialMatch,
648+
@Nonnull final TranslationMap topToTopTranslationMap,
642649
@Nonnull final Set<RequestedOrdering> requestedOrderings) {
643-
final var maxMatchMap = partialMatch.getMatchInfo().getMaxMatchMap();
644-
final var translationMapOptional =
645-
maxMatchMap.pullUpMaybe(Quantifier.current(), Quantifier.current());
646-
if (translationMapOptional.isEmpty()) {
647-
return Optional.empty();
648-
}
649-
final var translationMap = translationMapOptional.get();
650-
651650
boolean seenForward = false;
652651
boolean seenReverse = false;
653652
final var satisfyingRequestedOrderings = ImmutableSet.<RequestedOrdering>builder();
654653
for (final var requestedOrdering : requestedOrderings) {
655654
final var translatedRequestedOrdering =
656-
requestedOrdering.translateCorrelations(translationMap, true);
655+
requestedOrdering.translateCorrelations(topToTopTranslationMap, true);
657656

658657
final var scanDirectionForRequestedOrderingOptional =
659658
satisfiesRequestedOrdering(partialMatch, translatedRequestedOrdering);
@@ -1041,11 +1040,6 @@ private static Ordering orderingFromOrderingParts(final @Nonnull List<MatchedOrd
10411040
protected static boolean isCompatibleComparisonKey(@Nonnull Collection<Value> comparisonKeyValues,
10421041
@Nonnull List<Value> commonPrimaryKeyValues,
10431042
@Nonnull ImmutableSet<Value> equalityBoundKeyValues) {
1044-
// if (comparisonKeyValues.isEmpty()) {
1045-
// // everything is in one row
1046-
// return true;
1047-
// }
1048-
10491043
return commonPrimaryKeyValues
10501044
.stream()
10511045
.filter(commonPrimaryKeyValue -> !equalityBoundKeyValues.contains(commonPrimaryKeyValue))
@@ -1085,7 +1079,7 @@ private static void updateIntersectionInfoMap(@Nonnull final Map<BitSet, Interse
10851079
@Nonnull final IntersectionResult intersectionResult) {
10861080
Verify.verify(partition.size() >= 2);
10871081
final var cacheKey = intersectionInfoKey(partition);
1088-
if (intersectionResult.hasCommonIntersectionOrdering()) {
1082+
if (intersectionResult.hasViableIntersection()) {
10891083
if (!intersectionResult.getExpressions().isEmpty()) {
10901084
// This loop loops partition.size() times
10911085
for (final var subPartition : ChooseK.chooseK(partition, partition.size() - 1)) {
@@ -1137,6 +1131,8 @@ protected static class SingleMatchedAccess {
11371131
private final CorrelationIdentifier candidateTopAlias;
11381132
private final boolean reverseScanOrder;
11391133
@Nonnull
1134+
private final TranslationMap topToTopTranslationMap;
1135+
@Nonnull
11401136
private final Set<RequestedOrdering> satisfyingRequestedOrderings;
11411137
@Nonnull
11421138
private final Supplier<GroupByMappings> pulledUpGroupByMappingsSupplier;
@@ -1145,12 +1141,14 @@ public SingleMatchedAccess(@Nonnull final PartialMatch partialMatch,
11451141
@Nonnull final Compensation compensation,
11461142
@Nonnull final CorrelationIdentifier candidateTopAlias,
11471143
final boolean reverseScanOrder,
1144+
@Nonnull final TranslationMap topToTopTranslationMap,
11481145
@Nonnull final Set<RequestedOrdering> satisfyingRequestedOrderings) {
11491146
this.partialMatch = partialMatch;
11501147
this.compensation = compensation;
11511148
this.candidateTopAlias = candidateTopAlias;
11521149
this.reverseScanOrder = reverseScanOrder;
11531150
this.satisfyingRequestedOrderings = ImmutableSet.copyOf(satisfyingRequestedOrderings);
1151+
this.topToTopTranslationMap = topToTopTranslationMap;
11541152
this.pulledUpGroupByMappingsSupplier =
11551153
Suppliers.memoize(() -> partialMatch.getMatchInfo()
11561154
.adjustGroupByMappings(Quantifier.current(), partialMatch.getCandidateRef().get()));
@@ -1175,6 +1173,11 @@ public boolean isReverseScanOrder() {
11751173
return reverseScanOrder;
11761174
}
11771175

1176+
@Nonnull
1177+
public TranslationMap getTopToTopTranslationMap() {
1178+
return topToTopTranslationMap;
1179+
}
1180+
11781181
@Nonnull
11791182
public Set<RequestedOrdering> getSatisfyingRequestedOrderings() {
11801183
return satisfyingRequestedOrderings;
@@ -1262,7 +1265,7 @@ public List<RelationalExpression> getExpressions() {
12621265
return Objects.requireNonNull(expressions);
12631266
}
12641267

1265-
public boolean hasCommonIntersectionOrdering() {
1268+
public boolean hasViableIntersection() {
12661269
return commonIntersectionOrdering != null;
12671270
}
12681271

@@ -1277,7 +1280,7 @@ public Compensation getCompensation() {
12771280
}
12781281

12791282
@Nonnull
1280-
public static IntersectionResult noCommonOrdering() {
1283+
public static IntersectionResult noViableIntersection() {
12811284
return new IntersectionResult(null, Compensation.noCompensation(), ImmutableList.of());
12821285
}
12831286

0 commit comments

Comments
 (0)