add minimize to RecordQueryPlan (#3356)

Author: normen662

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/bitmap/ComposedBitmapIndexQueryPlan.java

@@ -47,6 +47,7 @@
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryCoveringIndexPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan;
 import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlanWithNoChildren;
+import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableList;
 import com.google.protobuf.Message;
 
@@ -206,6 +207,28 @@ public ComposedBitmapIndexQueryPlan translateCorrelations(@Nonnull final Transla
         return new ComposedBitmapIndexQueryPlan(translatedIndexPlansBuilder.build(), composer);
     }
 
+    @Override
+    public boolean canBeMinimized() {
+        return indexPlans.stream().anyMatch(RecordQueryCoveringIndexPlan::canBeMinimized);
+    }
+
+    @Nonnull
+    @Override
+    public RecordQueryPlan minimize(@Nonnull final List<Quantifier.Physical> newQuantifiers) {
+        Verify.verify(newQuantifiers.isEmpty());
+
+        final var minimizedIndexPlansBuilder = ImmutableList.<RecordQueryCoveringIndexPlan>builder();
+        for (final var indexPlan : indexPlans) {
+            if (indexPlan.canBeMinimized()) {
+                minimizedIndexPlansBuilder.add(indexPlan.minimize(newQuantifiers));
+            } else {
+                minimizedIndexPlansBuilder.add(indexPlan);
+            }
+        }
+
+        return new ComposedBitmapIndexQueryPlan(minimizedIndexPlansBuilder.build(), composer);
+    }
+
     @Nonnull
     @Override
     public Value getResultValue() {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlanningCostModel.java

@@ -3,7 +3,7 @@
  *
  * This source file is part of the FoundationDB open source project
  *
- * Copyright 2015-2019 Apple Inc. and the FoundationDB project authors
+ * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@
 package com.apple.foundationdb.record.query.plan.cascades;
 
 import com.apple.foundationdb.annotation.API;
+import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings;
 import com.apple.foundationdb.record.PlanHashable;
 import com.apple.foundationdb.record.query.expressions.Comparisons;
 import com.apple.foundationdb.record.query.plan.QueryPlanner.IndexScanPreference;
@@ -62,6 +63,7 @@
  */
 @API(API.Status.EXPERIMENTAL)
 @SuppressWarnings("PMD.TooManyStaticImports")
+@SpotBugsSuppressWarnings("SE_COMPARATOR_SHOULD_BE_SERIALIZABLE")
 public class PlanningCostModel implements CascadesCostModel {
     @Nonnull
     private static final Set<Class<? extends RelationalExpression>> interestingPlanClasses =

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/RewritingCostModel.java

@@ -21,6 +21,7 @@
 package com.apple.foundationdb.record.query.plan.cascades;
 
 import com.apple.foundationdb.annotation.API;
+import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings;
 import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression;
 
@@ -30,6 +31,7 @@
  * Cost model for {@link PlannerPhase#REWRITING}. TODO To be fleshed out whe we have actual rules.
  */
 @API(API.Status.EXPERIMENTAL)
+@SpotBugsSuppressWarnings("SE_COMPARATOR_SHOULD_BE_SERIALIZABLE")
 public class RewritingCostModel implements CascadesCostModel {
     @Nonnull
     private final RecordQueryPlannerConfiguration configuration;

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/StableSelectorCostModel.java

@@ -0,0 +1,58 @@
+/*
+ * StableSelectorCostModel.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.record.query.plan.cascades;
+
+import com.apple.foundationdb.annotation.API;
+import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings;
+import com.apple.foundationdb.record.PlanHashable;
+import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration;
+import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression;
+
+import javax.annotation.Nonnull;
+
+/**
+ * A comparator implementing a simple cost model for the {@link CascadesPlanner} to choose the plan with the smallest
+ * plan hash.
+ */
+@API(API.Status.EXPERIMENTAL)
+@SpotBugsSuppressWarnings("SE_COMPARATOR_SHOULD_BE_SERIALIZABLE")
+public class StableSelectorCostModel implements CascadesCostModel {
+    @Nonnull
+    @Override
+    public RecordQueryPlannerConfiguration getConfiguration() {
+        return RecordQueryPlannerConfiguration.defaultPlannerConfiguration();
+    }
+
+    @Override
+    public int compare(@Nonnull final RelationalExpression a, @Nonnull final RelationalExpression b) {
+        //
+        // If plans are indistinguishable from a cost perspective, select one by planHash. This makes the cost model
+        // stable (select the same plan on subsequent plannings).
+        //
+        if ((a instanceof PlanHashable) && (b instanceof PlanHashable)) {
+            int hA = ((PlanHashable)a).planHash(PlanHashable.CURRENT_FOR_CONTINUATION);
+            int hB = ((PlanHashable)b).planHash(PlanHashable.CURRENT_FOR_CONTINUATION);
+            return Integer.compare(hA, hB);
+        }
+
+        return 0;
+    }
+}

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryAggregateIndexPlan.java

@@ -271,6 +271,19 @@ public RecordQueryAggregateIndexPlan translateCorrelations(@Nonnull final Transl
         return this;
     }
 
+    @Override
+    public boolean canBeMinimized() {
+        return indexPlan.canBeMinimized();
+    }
+
+    @Nonnull
+    @Override
+    public RecordQueryPlan minimize(@Nonnull final List<Quantifier.Physical> newQuantifiers) {
+        Verify.verify(newQuantifiers.isEmpty());
+        return new RecordQueryAggregateIndexPlan(indexPlan.minimize(newQuantifiers), recordTypeName, toRecord,
+                resultValue, groupByResultValue, constraint);
+    }
+
     @Override
     @SuppressWarnings("PMD.CompareObjectsWithEquals")
     public boolean equalsWithoutChildren(@Nonnull RelationalExpression otherExpression,

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryCoveringIndexPlan.java

@@ -241,6 +241,19 @@ public RecordQueryCoveringIndexPlan translateCorrelations(@Nonnull final Transla
         return this;
     }
 
+    @Override
+    public boolean canBeMinimized() {
+        return indexPlan.canBeMinimized();
+    }
+
+    @Nonnull
+    @Override
+    public RecordQueryCoveringIndexPlan minimize(@Nonnull final List<Quantifier.Physical> newQuantifiers) {
+        Verify.verify(newQuantifiers.isEmpty());
+        return new RecordQueryCoveringIndexPlan((RecordQueryPlanWithIndex)indexPlan.minimize(newQuantifiers),
+                recordTypeName, getAvailableFields(), toRecord);
+    }
+
     @Nonnull
     public Optional<Value> pushValueThroughFetch(@Nonnull Value value,
                                                  @Nonnull CorrelationIdentifier sourceAlias,

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java

@@ -475,20 +475,26 @@ public RecordQueryIndexPlan translateCorrelations(@Nonnull final TranslationMap
         return withIndexScanParameters(scanParameters.translateCorrelations(translationMap, shouldSimplifyValues));
     }
 
+    @Override
+    public boolean canBeMinimized() {
+        return matchCandidateOptional.isPresent();
+    }
+
     @Nonnull
-    protected RecordQueryIndexPlan withIndexScanParameters(@Nonnull final IndexScanParameters newIndexScanParameters) {
-        return new RecordQueryIndexPlan(indexName,
-                commonPrimaryKey,
-                newIndexScanParameters,
-                indexFetchMethod,
-                fetchIndexRecords,
-                reverse,
-                strictlySorted,
-                matchCandidateOptional,
-                resultType,
+    @Override
+    public RecordQueryIndexPlan minimize(@Nonnull final List<Quantifier.Physical> newQuantifiers) {
+        Verify.verify(newQuantifiers.isEmpty());
+        return new RecordQueryIndexPlan(indexName, commonPrimaryKey, scanParameters, indexFetchMethod,
+                fetchIndexRecords, reverse, strictlySorted, Optional.empty(), resultType,
                 constraint);
     }
 
+    @Nonnull
+    protected RecordQueryIndexPlan withIndexScanParameters(@Nonnull final IndexScanParameters newIndexScanParameters) {
+        return new RecordQueryIndexPlan(indexName, commonPrimaryKey, newIndexScanParameters, indexFetchMethod,
+                fetchIndexRecords, reverse, strictlySorted, matchCandidateOptional, resultType, constraint);
+    }
+
     @Nonnull
     @Override
     public AvailableFields getAvailableFields() {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryPlan.java

@@ -22,7 +22,6 @@
 
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.annotation.GenerateVisitor;
-import com.apple.foundationdb.record.query.plan.HeuristicPlanner;
 import com.apple.foundationdb.record.EvaluationContext;
 import com.apple.foundationdb.record.ExecuteProperties;
 import com.apple.foundationdb.record.PlanSerializable;
@@ -33,7 +32,9 @@
 import com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase;
+import com.apple.foundationdb.record.query.combinatorics.TopologicalSort;
 import com.apple.foundationdb.record.query.plan.AvailableFields;
+import com.apple.foundationdb.record.query.plan.HeuristicPlanner;
 import com.apple.foundationdb.record.query.plan.cascades.AliasMap;
 import com.apple.foundationdb.record.query.plan.cascades.FinalMemoizer;
 import com.apple.foundationdb.record.query.plan.cascades.Quantifier;
@@ -44,13 +45,18 @@
 import com.apple.foundationdb.record.query.plan.serialization.PlanSerialization;
 import com.apple.foundationdb.record.query.plan.visitor.RecordQueryPlannerSubstitutionVisitor;
 import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
 import com.google.protobuf.Message;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
+import static com.apple.foundationdb.record.query.plan.cascades.properties.ReferencesAndDependenciesProperty.referencesAndDependencies;
+
 /**
  * An executable query plan for producing records.
  * <br>
@@ -163,6 +169,87 @@ default RecordQueryPlan strictlySorted(@Nonnull FinalMemoizer memoizer) {
         return this;
     }
 
+    /**
+     * Returns whether a plan with a smaller footprint can be generated by removing references to e.g. planning
+     * bread crumbs such as match candidates, etc.
+     * @return {@code true} iff the plan can be minimized.
+     */
+    default boolean canBeMinimized() {
+        return false;
+    }
+
+    /**
+     * Minimizes the plan by removing extraneous references to objects that are not strictly needed to execute the
+     * plan.
+     * @return a minimized plan
+     */
+    @Nonnull
+    @SuppressWarnings("PMD.CompareObjectsWithEquals")
+    default RecordQueryPlan minimize() {
+        //
+        // We need to order all references in a way that only references occurring later can be dependent on references
+        // occurring earlier.
+        //
+        final var referenceDependencyOrder =
+                referencesAndDependencies().evaluate(this);
+
+        final var references =
+                TopologicalSort.anyTopologicalOrderPermutation(referenceDependencyOrder)
+                        .orElseThrow(() -> new RecordCoreException("graph has cycles"));
+
+        //
+        // We keep this cache to make sure that CSEs prior to minimization end up being CSEs after. We don't want to
+        // break them as would be the case with a straightforward recursive approach.
+        //
+        final var minimizationCache =
+                Maps.<Reference, Reference>newIdentityHashMap();
+
+        for (final var reference : references) {
+            final var plan = reference.getOnlyElementAsPlan();
+
+            final var minimizedPlan =
+                    minimizePlanOverMinimizedReferences(minimizationCache, reference.getOnlyElementAsPlan());
+
+            if (minimizedPlan != plan) {
+                minimizationCache.put(reference, Reference.plannedOf(minimizedPlan));
+            }
+        }
+
+        return minimizePlanOverMinimizedReferences(minimizationCache, this);
+    }
+
+    @Nonnull
+    default RecordQueryPlan minimize(@Nonnull final List<Quantifier.Physical> newQuantifiers) {
+        throw new UnsupportedOperationException("RecordQueryPlan by default cannot be minimized");
+    }
+
+    @Nonnull
+    private static RecordQueryPlan minimizePlanOverMinimizedReferences(@Nonnull final Map<Reference, Reference> minimizationCache,
+                                                                       @Nonnull final RecordQueryPlan plan) {
+        var allMinimizedChildrenSame = true;
+        final var newQuantifiersBuilder = ImmutableList.<Quantifier.Physical>builder();
+        for (final var quantifier : plan.getQuantifiers()) {
+            final var physicalQuantifier = quantifier.narrow(Quantifier.Physical.class);
+            final var childReference = physicalQuantifier.getRangesOver();
+
+            final Reference translatedChildReference = minimizationCache.get(childReference);
+            if (translatedChildReference != null) {
+                newQuantifiersBuilder.add(physicalQuantifier.overNewReference(translatedChildReference));
+                allMinimizedChildrenSame = false;
+            } else {
+                newQuantifiersBuilder.add(physicalQuantifier);
+            }
+        }
+
+        if (plan.canBeMinimized()) {
+            return plan.minimize(newQuantifiersBuilder.build());
+        }
+        if (allMinimizedChildrenSame) {
+            return plan;
+        }
+        return (RecordQueryPlan)plan.withQuantifiers(newQuantifiersBuilder.build());
+    }
+
     // we know the type of the group, even though the compiler doesn't, intentional use of reference equality
     @HeuristicPlanner
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryScanPlan.java

@@ -315,6 +315,19 @@ public RecordQueryScanPlan translateCorrelations(@Nonnull final TranslationMap t
                 matchCandidateOptional);
     }
 
+    @Override
+    public boolean canBeMinimized() {
+        return matchCandidateOptional.isPresent();
+    }
+
+    @Nonnull
+    @Override
+    public RecordQueryPlan minimize(@Nonnull final List<Quantifier.Physical> newQuantifiers) {
+        Verify.verify(newQuantifiers.isEmpty());
+        return new RecordQueryScanPlan(recordTypes, flowedType, commonPrimaryKey, comparisons, reverse, strictlySorted,
+                Optional.empty());
+    }
+
     @Nonnull
     @Override
     public Value getResultValue() {

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerInvokedRoutine.java

@@ -25,7 +25,6 @@
 import com.apple.foundationdb.relational.recordlayer.query.functions.CompiledSqlFunction;
 import com.apple.foundationdb.relational.util.Assert;
 import com.google.common.base.Supplier;
-import com.google.common.base.Suppliers;
 
 import javax.annotation.Nonnull;
 import java.util.Objects;
@@ -46,7 +45,8 @@ public RecordLayerInvokedRoutine(@Nonnull final String description,
                                      @Nonnull final Supplier<CompiledSqlFunction> compilableSqlFunctionSupplier) {
         this.description = description;
         this.name = name;
-        this.compilableSqlFunctionSupplier = Suppliers.memoize(compilableSqlFunctionSupplier);
+        // TODO this used to be memoized
+        this.compilableSqlFunctionSupplier = compilableSqlFunctionSupplier;
     }
 
     @Nonnull