Skip to content

Commit 28ed247

Browse files
normen662normen662
committed
agg testcases run
1 parent aa4fd61 commit 28ed247

File tree

9 files changed

+195
-123
lines changed

9 files changed

+195
-123
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexMatchCandidate.java

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -207,8 +207,8 @@ private int getPermutedCount() {
207207
public List<MatchedOrderingPart> computeMatchedOrderingParts(@Nonnull final MatchInfo matchInfo,
208208
@Nonnull final List<CorrelationIdentifier> sortParameterIds,
209209
final boolean isReverse) {
210-
final var parameterBindingMap =
211-
matchInfo.getRegularMatchInfo().getParameterBindingMap();
210+
final var regularMatchInfo = matchInfo.getRegularMatchInfo();
211+
final var parameterBindingMap = regularMatchInfo.getParameterBindingMap();
212212
final var normalizedKeyExpressions =
213213
getFullKeyExpression().normalizeKeyForPositions();
214214

@@ -220,13 +220,17 @@ public List<MatchedOrderingPart> computeMatchedOrderingParts(@Nonnull final Matc
220220
final var deconstructedValues =
221221
Values.deconstructRecord(QuantifiedObjectValue.of(Quantifier.current(), selectHavingResultType));
222222

223+
final var rollUpToGroupingValues = regularMatchInfo.getRollUpToGroupingValues();
224+
final int orderSize = rollUpToGroupingValues != null ? rollUpToGroupingValues.size() : sortParameterIds.size();
225+
223226
// Compute the ordering for this index by collecting the result values of the selectHaving statement
224227
// associated with each sortParameterId. Note that for most aggregate indexes, the aggregate value is
225228
// in the FDB value, and so it does not contribute to the ordering of the index, so there is no sortParameterId
226229
// corresponding to it. For the PERMUTED_MIN and PERMUTED_MAX indexes, the aggregate _does_ have a corresponding
227230
// sortParameterId. Its position is determined by the permutedSize option, handled below by adjusting the
228231
// sortParameterId's index before looking it up in the original key expression
229-
for (final var parameterId : sortParameterIds) {
232+
for (int i = 0; i < orderSize; i++) {
233+
final var parameterId = sortParameterIds.get(i);
230234
final var ordinalInCandidate = candidateParameterIds.indexOf(parameterId);
231235
Verify.verify(ordinalInCandidate >= 0);
232236
int permutedIndex = indexWithPermutation(ordinalInCandidate);
@@ -440,6 +444,7 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch
440444
selectHavingResultValue,
441445
groupByResultValue,
442446
constraintMaybe);
447+
443448
if (regularMatchInfo.getRollUpToGroupingValues() != null) {
444449
//
445450
// We need to perform a roll up.
@@ -449,14 +454,13 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch
449454

450455
//final var recordValues = Values.deconstructRecord(recordValue);
451456
final var groupingAndAggregateAccessors =
452-
getGroupingAndAggregateAccessors(regularMatchInfo.getRollUpToGroupingValues().size(),
453-
aggregateIndexScanAlias);
457+
getGroupingAndAggregateAccessors(getGroupingCount(), aggregateIndexScanAlias);
454458
final var groupingAccessorValues = groupingAndAggregateAccessors.getLeft();
455459
final var aggregateAccessorValue = groupingAndAggregateAccessors.getRight();
456460
final var allFields = resultType.getFields();
457461
final var rollUpGroupingColumnsBuilder =
458462
ImmutableList.<Column<? extends Value>>builder();
459-
for (int i = 0; i < groupingAccessorValues.size(); i++) {
463+
for (int i = 0; i < regularMatchInfo.getRollUpToGroupingValues().size(); i++) {
460464
final var field = allFields.get(i);
461465
rollUpGroupingColumnsBuilder.add(Column.of(field, groupingAccessorValues.get(i)));
462466
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Compensation.java

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -639,6 +639,8 @@ default Compensation intersect(@Nonnull Compensation otherCompensation) {
639639
final var newGroupByMappings = GroupByMappings.of(newMatchedGroupingsMapBuilder.build(),
640640
newMatchedAggregatesMap, newUnmatchedAggregatesMapBuilder.build());
641641

642+
boolean isImpossible = false;
643+
642644
final ResultCompensationFunction newResultResultCompensationFunction;
643645
final var resultCompensationFunction = getResultCompensationFunction();
644646
final var otherResultCompensationFunction = otherWithSelectCompensation.getResultCompensationFunction();
@@ -651,6 +653,7 @@ default Compensation intersect(@Nonnull Compensation otherCompensation) {
651653
// pick the one from this side -- it does not matter as both candidates have the same shape
652654
newResultResultCompensationFunction =
653655
resultCompensationFunction.amend(unmatchedAggregateMap, newMatchedAggregatesMap);
656+
isImpossible |= newResultResultCompensationFunction.isImpossible();
654657
}
655658

656659
final var otherCompensationMap =
@@ -661,14 +664,17 @@ default Compensation intersect(@Nonnull Compensation otherCompensation) {
661664
final var otherPredicateCompensationFunction = otherCompensationMap.get(entry.getKey());
662665
if (otherPredicateCompensationFunction != null) {
663666
// Both compensations have a compensation for this particular predicate which is essentially
664-
// reapplying the predicate. Three cases arise:
667+
// reapplying the predicate. Two cases arise:
665668
// 1. Both predicate compensation functions are needed and possible. At this point it doesn't
666669
// matter which side we take as both create the same compensating filter. If at any point in the
667670
// future one data access has a better reapplication we need to generate plan variants with
668671
// either compensation and let the planner figure out which one wins. We just pick one side here.
669-
// 2. TODO.
670-
combinedPredicateMap.put(entry.getKey(),
671-
entry.getValue().amend(unmatchedAggregateMap, newMatchedAggregatesMap));
672+
// 2. Either one or both compensation functions are impossible, but thee intersection is possible.
673+
// We take the compensation function from this side and amend it with the compensation function
674+
// from the other side.
675+
final var newPredicateCompensationFunction = entry.getValue().amend(unmatchedAggregateMap, newMatchedAggregatesMap);
676+
combinedPredicateMap.put(entry.getKey(), newPredicateCompensationFunction);
677+
isImpossible |= newPredicateCompensationFunction.isImpossible();
672678
}
673679
}
674680

@@ -691,10 +697,12 @@ default Compensation intersect(@Nonnull Compensation otherCompensation) {
691697

692698
final var unmatchedQuantifiers = Sets.intersection(this.getUnmatchedQuantifiers(), otherWithSelectCompensation.getUnmatchedQuantifiers());
693699
final var unmatchedQuantifierAliases = unmatchedQuantifiers.stream().map(Quantifier::getAlias).collect(ImmutableList.toImmutableList());
694-
final var isImpossible = combinedPredicateMap.keySet()
695-
.stream()
696-
.flatMap(queryPredicate -> queryPredicate.getCorrelatedTo().stream())
697-
.anyMatch(unmatchedQuantifierAliases::contains);
700+
if (!isImpossible) {
701+
isImpossible = combinedPredicateMap.keySet()
702+
.stream()
703+
.flatMap(queryPredicate -> queryPredicate.getCorrelatedTo().stream())
704+
.anyMatch(unmatchedQuantifierAliases::contains);
705+
}
698706

699707
return intersectedChildCompensation.derived(isImpossible,
700708
combinedPredicateMap,

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -518,8 +518,10 @@ public Ordering visitInUnionOnKeyExpressionPlan(@Nonnull final RecordQueryInUnio
518518

519519
@Nonnull
520520
@Override
521-
public Ordering visitMultiIntersectionOnValuesPlan(@Nonnull final RecordQueryMultiIntersectionOnValuesPlan element) {
522-
return Ordering.empty(); // TODO
521+
public Ordering visitMultiIntersectionOnValuesPlan(@Nonnull final RecordQueryMultiIntersectionOnValuesPlan multiIntersectionOnValuesPlan) {
522+
final var orderings = orderingsFromChildren(multiIntersectionOnValuesPlan);
523+
return deriveForDistinctSetOperationFromOrderings(orderings,
524+
multiIntersectionOnValuesPlan.getComparisonKeyOrderingParts(), Ordering.INTERSECTION);
523525
}
524526

525527
@Nonnull

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java

Lines changed: 33 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,7 @@ protected Set<? extends RelationalExpression> dataAccessForMatchPartition(@Nonnu
337337
bestMatchToDistinctPlanMap,
338338
binaryPartition,
339339
requestedOrderings);
340-
if (binaryIntersections.hasCommonIntersectionOrdering()) {
340+
if (binaryIntersections.hasViableIntersection()) {
341341
updateIntersectionInfoMap(intersectionInfoMap, binaryPartition, binaryIntersections);
342342
} else {
343343
if (sieveBitMatrix != null) {
@@ -395,7 +395,10 @@ protected Set<? extends RelationalExpression> dataAccessForMatchPartition(@Nonnu
395395
kPartition,
396396
requestedOrderings);
397397

398-
Verify.verify(intersectionResult.hasCommonIntersectionOrdering());
398+
if (!intersectionResult.hasViableIntersection()) {
399+
continue;
400+
}
401+
399402
hasCommonOrderingForK = true;
400403

401404
updateIntersectionInfoMap(intersectionInfoMap, kPartition, intersectionResult);
@@ -432,14 +435,6 @@ protected static Optional<List<Value>> commonRecordKeyValuesMaybe(@Nonnull Itera
432435
}
433436
key = keyMaybe.get();
434437
} else if (matchCandidate instanceof AggregateIndexMatchCandidate) {
435-
// final var aggregateIndexMatchCandidate = (AggregateIndexMatchCandidate)matchCandidate;
436-
// final var rollUpToGroupingValues = regularMatchInfo.getRollUpToGroupingValues();
437-
// if (rollUpToGroupingValues == null) {
438-
// key = aggregateIndexMatchCandidate.getGroupByValues();
439-
// } else {
440-
// key = aggregateIndexMatchCandidate.getGroupByValues().subList(0, rollUpToGroupingValues.size());
441-
// }
442-
443438
final var aggregateIndexMatchCandidate = (AggregateIndexMatchCandidate)matchCandidate;
444439
final var rollUpToGroupingValues = regularMatchInfo.getRollUpToGroupingValues();
445440
if (rollUpToGroupingValues == null) {
@@ -582,8 +577,13 @@ private static List<SingleMatchedAccess> prepareMatchesAndCompensations(final @N
582577
final @Nonnull Set<RequestedOrdering> requestedOrderings) {
583578
final var partialMatchesWithCompensation = new ArrayList<SingleMatchedAccess>();
584579
for (final var partialMatch: partialMatches) {
580+
final var topToTopTranslationMapOptional = computeTopToTopTranslationMapMaybe(partialMatch);
581+
if (topToTopTranslationMapOptional.isEmpty()) {
582+
continue;
583+
}
584+
final var topToTopTranslationMap = topToTopTranslationMapOptional.get();
585585
final var satisfyingOrderingsPairOptional =
586-
satisfiesAnyRequestedOrderings(partialMatch, requestedOrderings);
586+
satisfiesAnyRequestedOrderings(partialMatch, topToTopTranslationMap, requestedOrderings);
587587
if (satisfyingOrderingsPairOptional.isEmpty()) {
588588
continue;
589589
}
@@ -603,7 +603,7 @@ private static List<SingleMatchedAccess> prepareMatchesAndCompensations(final @N
603603

604604
if (scanDirection == ScanDirection.FORWARD || scanDirection == ScanDirection.BOTH) {
605605
partialMatchesWithCompensation.add(new SingleMatchedAccess(partialMatch, compensation,
606-
topAlias, false, satisfyingOrderingsPair.getRight()));
606+
topAlias, false, topToTopTranslationMap, satisfyingOrderingsPair.getRight()));
607607
}
608608

609609
//
@@ -616,7 +616,7 @@ private static List<SingleMatchedAccess> prepareMatchesAndCompensations(final @N
616616
//
617617
if (scanDirection == ScanDirection.REVERSE /* || scanDirection == ScanDirection.BOTH */) {
618618
partialMatchesWithCompensation.add(new SingleMatchedAccess(partialMatch, compensation,
619-
topAlias, true, satisfyingOrderingsPair.getRight()));
619+
topAlias, true, topToTopTranslationMap, satisfyingOrderingsPair.getRight()));
620620
}
621621
}
622622

@@ -626,6 +626,12 @@ private static List<SingleMatchedAccess> prepareMatchesAndCompensations(final @N
626626
return partialMatchesWithCompensation;
627627
}
628628

629+
@Nonnull
630+
private static Optional<TranslationMap> computeTopToTopTranslationMapMaybe(final PartialMatch partialMatch) {
631+
final var maxMatchMap = partialMatch.getMatchInfo().getMaxMatchMap();
632+
return maxMatchMap.pullUpMaybe(Quantifier.current(), Quantifier.current());
633+
}
634+
629635
/**
630636
* Private helper method to compute the subset of orderings passed in that would be satisfied by a scan
631637
* if the given {@link PartialMatch} were to be planned.
@@ -638,21 +644,14 @@ private static List<SingleMatchedAccess> prepareMatchesAndCompensations(final @N
638644
@Nonnull
639645
@SuppressWarnings("java:S135")
640646
private static Optional<NonnullPair<ScanDirection, Set<RequestedOrdering>>> satisfiesAnyRequestedOrderings(@Nonnull final PartialMatch partialMatch,
647+
@Nonnull final TranslationMap topToTopTranslationMap,
641648
@Nonnull final Set<RequestedOrdering> requestedOrderings) {
642-
final var maxMatchMap = partialMatch.getMatchInfo().getMaxMatchMap();
643-
final var translationMapOptional =
644-
maxMatchMap.pullUpMaybe(Quantifier.current(), Quantifier.current());
645-
if (translationMapOptional.isEmpty()) {
646-
return Optional.empty();
647-
}
648-
final var translationMap = translationMapOptional.get();
649-
650649
boolean seenForward = false;
651650
boolean seenReverse = false;
652651
final var satisfyingRequestedOrderings = ImmutableSet.<RequestedOrdering>builder();
653652
for (final var requestedOrdering : requestedOrderings) {
654653
final var translatedRequestedOrdering =
655-
requestedOrdering.translateCorrelations(translationMap, true);
654+
requestedOrdering.translateCorrelations(topToTopTranslationMap, true);
656655

657656
final var scanDirectionForRequestedOrderingOptional =
658657
satisfiesRequestedOrdering(partialMatch, translatedRequestedOrdering);
@@ -1040,11 +1039,6 @@ private static Ordering orderingFromOrderingParts(final @Nonnull List<MatchedOrd
10401039
protected static boolean isCompatibleComparisonKey(@Nonnull Collection<Value> comparisonKeyValues,
10411040
@Nonnull List<Value> commonPrimaryKeyValues,
10421041
@Nonnull ImmutableSet<Value> equalityBoundKeyValues) {
1043-
// if (comparisonKeyValues.isEmpty()) {
1044-
// // everything is in one row
1045-
// return true;
1046-
// }
1047-
10481042
return commonPrimaryKeyValues
10491043
.stream()
10501044
.filter(commonPrimaryKeyValue -> !equalityBoundKeyValues.contains(commonPrimaryKeyValue))
@@ -1084,7 +1078,7 @@ private static void updateIntersectionInfoMap(@Nonnull final Map<BitSet, Interse
10841078
@Nonnull final IntersectionResult intersectionResult) {
10851079
Verify.verify(partition.size() >= 2);
10861080
final var cacheKey = intersectionInfoKey(partition);
1087-
if (intersectionResult.hasCommonIntersectionOrdering()) {
1081+
if (intersectionResult.hasViableIntersection()) {
10881082
if (!intersectionResult.getExpressions().isEmpty()) {
10891083
// This loop loops partition.size() times
10901084
for (final var subPartition : ChooseK.chooseK(partition, partition.size() - 1)) {
@@ -1136,6 +1130,8 @@ protected static class SingleMatchedAccess {
11361130
private final CorrelationIdentifier candidateTopAlias;
11371131
private final boolean reverseScanOrder;
11381132
@Nonnull
1133+
private final TranslationMap topToTopTranslationMap;
1134+
@Nonnull
11391135
private final Set<RequestedOrdering> satisfyingRequestedOrderings;
11401136
@Nonnull
11411137
private final Supplier<GroupByMappings> pulledUpGroupByMappingsSupplier;
@@ -1144,12 +1140,14 @@ public SingleMatchedAccess(@Nonnull final PartialMatch partialMatch,
11441140
@Nonnull final Compensation compensation,
11451141
@Nonnull final CorrelationIdentifier candidateTopAlias,
11461142
final boolean reverseScanOrder,
1143+
@Nonnull final TranslationMap topToTopTranslationMap,
11471144
@Nonnull final Set<RequestedOrdering> satisfyingRequestedOrderings) {
11481145
this.partialMatch = partialMatch;
11491146
this.compensation = compensation;
11501147
this.candidateTopAlias = candidateTopAlias;
11511148
this.reverseScanOrder = reverseScanOrder;
11521149
this.satisfyingRequestedOrderings = ImmutableSet.copyOf(satisfyingRequestedOrderings);
1150+
this.topToTopTranslationMap = topToTopTranslationMap;
11531151
this.pulledUpGroupByMappingsSupplier =
11541152
Suppliers.memoize(() -> partialMatch.getMatchInfo()
11551153
.adjustGroupByMappings(Quantifier.current(), partialMatch.getCandidateRef().get()));
@@ -1174,6 +1172,11 @@ public boolean isReverseScanOrder() {
11741172
return reverseScanOrder;
11751173
}
11761174

1175+
@Nonnull
1176+
public TranslationMap getTopToTopTranslationMap() {
1177+
return topToTopTranslationMap;
1178+
}
1179+
11771180
@Nonnull
11781181
public Set<RequestedOrdering> getSatisfyingRequestedOrderings() {
11791182
return satisfyingRequestedOrderings;
@@ -1261,7 +1264,7 @@ public List<RelationalExpression> getExpressions() {
12611264
return Objects.requireNonNull(expressions);
12621265
}
12631266

1264-
public boolean hasCommonIntersectionOrdering() {
1267+
public boolean hasViableIntersection() {
12651268
return commonIntersectionOrdering != null;
12661269
}
12671270

@@ -1276,7 +1279,7 @@ public Compensation getCompensation() {
12761279
}
12771280

12781281
@Nonnull
1279-
public static IntersectionResult noCommonOrdering() {
1282+
public static IntersectionResult noViableIntersection() {
12801283
return new IntersectionResult(null, Compensation.noCompensation(), ImmutableList.of());
12811284
}
12821285

0 commit comments

Comments
 (0)