Skip to content

Commit 7264c08

Browse files
hatyohatyo
authored
Support SQL Table Valued Functions (#3280)
This provides support to table-valued-functions, it introduces the necessary framework support across different components: parsing, semantic analysis, plan generation, planning, and execution, to pre-defined table-valued functions. Two pre-defined table-functions are introduced in this PR: #### 1. `values` function: Most major SQL vendors support this function, the relational engine now supports this function as well: ```sql select * from values (1, 'a', 3.0), (4, 'b', 4.5); -- returns two rows: -- 1, 'a', 3.0 -- 4, 'b', 4.5 ``` It also enables naming the resulting table along with its columns: ```sql select * from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) -- returns two rows with the following metadata: -- B | C | W(X, Y, Z) -- 1 | 2.0 | (3, 4.0, 'foo') -- 10 | 90.2 | (5, 6.0, 'bar') ``` Notice how the value `4` in the first row's nested structure `W.Y` was promoted to `4.0`, this is because this function analyzes all the rows, and for each leaf in every (nested) tuple, it looks for a type that accommodates for all the individual values of the leaf across all rows (max type) and injects type promotions if necessary. Also note that the type aliasing works with nested array and structs, to some degrees addresses some of the issues reported in #3231, and in the future, the new logic can be leveraged to resolve that ticket as well. #### 2. `range` function: This is also a very common function, it returns a range between: - given start (inclusive), defaulting to `0`. - given end (exclusive). - optional step size. - ```sql select ID from range (1, 5) -- returns 4 rows -- 1 -- 2 -- 3 -- 4 ``` The implementation of this TVF has streaming semantics. In other words, it does not pre-materialize all the values in the range, but rather return them on-demand. With this PR, we can hopefully start adding more advanced TVFs easily. This resolves #3282.
1 parent 3f82672 commit 7264c08

37 files changed

+1952
-51
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* BuiltInTableFunction.java
3+
*
4+
* This source file is part of the FoundationDB open source project
5+
*
6+
* Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
package com.apple.foundationdb.record.query.plan.cascades;
22+
23+
import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
24+
import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue;
25+
26+
import javax.annotation.Nonnull;
27+
import javax.annotation.Nullable;
28+
import java.util.List;
29+
30+
public abstract class BuiltInTableFunction extends BuiltInFunction<StreamingValue> {
31+
protected BuiltInTableFunction(@Nonnull final String functionName,
32+
@Nonnull final List<Type> parameterTypes,
33+
@Nonnull final EncapsulationFunction<StreamingValue> encapsulationFunction) {
34+
super(functionName, parameterTypes, encapsulationFunction);
35+
}
36+
37+
protected BuiltInTableFunction(@Nonnull final String functionName, @Nonnull final List<Type> parameterTypes,
38+
@Nullable final Type variadicSuffixType, @Nonnull final EncapsulationFunction<StreamingValue> encapsulationFunction) {
39+
super(functionName, parameterTypes, variadicSuffixType, encapsulationFunction);
40+
}
41+
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementInUnionRule;
3636
import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementInsertRule;
3737
import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementRecursiveUnionRule;
38+
import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementTableFunctionRule;
3839
import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementTempTableInsertRule;
3940
import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementIntersectionRule;
4041
import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementNestedLoopJoinRule;
@@ -179,7 +180,8 @@ public class PlannerRuleSet {
179180
new ImplementInsertRule(),
180181
new ImplementTempTableInsertRule(),
181182
new ImplementUpdateRule(),
182-
new ImplementRecursiveUnionRule()
183+
new ImplementRecursiveUnionRule(),
184+
new ImplementTableFunctionRule()
183185
);
184186

185187
private static final List<CascadesRule<? extends RelationalExpression>> EXPLORATION_RULES =

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
/*
2+
* TableFunctionExpression.java
3+
*
4+
* This source file is part of the FoundationDB open source project
5+
*
6+
* Copyright 2015-2020 Apple Inc. and the FoundationDB project authors
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
package com.apple.foundationdb.record.query.plan.cascades.expressions;
22+
23+
import com.apple.foundationdb.annotation.API;
24+
import com.apple.foundationdb.record.EvaluationContext;
25+
import com.apple.foundationdb.record.query.plan.cascades.AliasMap;
26+
import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange;
27+
import com.apple.foundationdb.record.query.plan.cascades.Compensation;
28+
import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
29+
import com.apple.foundationdb.record.query.plan.cascades.IdentityBiMap;
30+
import com.apple.foundationdb.record.query.plan.cascades.MatchInfo;
31+
import com.apple.foundationdb.record.query.plan.cascades.PartialMatch;
32+
import com.apple.foundationdb.record.query.plan.cascades.Quantifier;
33+
import com.apple.foundationdb.record.query.plan.cascades.explain.InternalPlannerGraphRewritable;
34+
import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph;
35+
import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
36+
import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue;
37+
import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue;
38+
import com.apple.foundationdb.record.query.plan.cascades.values.Value;
39+
import com.apple.foundationdb.record.query.plan.cascades.values.translation.PullUp;
40+
import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap;
41+
import com.google.common.collect.ImmutableList;
42+
import com.google.common.collect.ImmutableMap;
43+
44+
import javax.annotation.Nonnull;
45+
import javax.annotation.Nullable;
46+
import java.util.Collections;
47+
import java.util.List;
48+
import java.util.Map;
49+
import java.util.Objects;
50+
import java.util.Set;
51+
52+
/**
53+
* A table function expression that delegates the actual execution to an underlying {@link StreamingValue} which
54+
* effectively returns a stream of results.
55+
*/
56+
@API(API.Status.EXPERIMENTAL)
57+
public class TableFunctionExpression implements RelationalExpression, InternalPlannerGraphRewritable {
58+
@Nonnull
59+
private final StreamingValue value;
60+
61+
public TableFunctionExpression(@Nonnull final StreamingValue value) {
62+
this.value = value;
63+
}
64+
65+
@Nonnull
66+
@Override
67+
public Value getResultValue() {
68+
return new QueriedValue(value.getResultType());
69+
}
70+
71+
@Nonnull
72+
@Override
73+
public Set<Type> getDynamicTypes() {
74+
return value.getDynamicTypes();
75+
}
76+
77+
@Nonnull
78+
public StreamingValue getValue() {
79+
return value;
80+
}
81+
82+
@Nonnull
83+
@Override
84+
public List<? extends Quantifier> getQuantifiers() {
85+
return Collections.emptyList();
86+
}
87+
88+
@Nonnull
89+
@Override
90+
public Set<CorrelationIdentifier> getCorrelatedTo() {
91+
return value.getCorrelatedTo();
92+
}
93+
94+
@Override
95+
@SuppressWarnings("PMD.CompareObjectsWithEquals")
96+
public boolean equalsWithoutChildren(@Nonnull RelationalExpression otherExpression,
97+
@Nonnull final AliasMap equivalencesMap) {
98+
if (this == otherExpression) {
99+
return true;
100+
}
101+
if (!(otherExpression instanceof TableFunctionExpression)) {
102+
return false;
103+
}
104+
105+
final var otherTableFunctionExpression = (TableFunctionExpression)otherExpression;
106+
107+
return value.semanticEquals(otherTableFunctionExpression.getValue(), equivalencesMap);
108+
}
109+
110+
@Override
111+
public int hashCodeWithoutChildren() {
112+
return Objects.hash(value);
113+
}
114+
115+
@Nonnull
116+
@Override
117+
@SuppressWarnings("PMD.CompareObjectsWithEquals")
118+
public TableFunctionExpression translateCorrelations(@Nonnull final TranslationMap translationMap,
119+
final boolean shouldSimplifyValues,
120+
@Nonnull final List<? extends Quantifier> translatedQuantifiers) {
121+
final Value translatedCollectionValue = value.translateCorrelations(translationMap, shouldSimplifyValues);
122+
if (translatedCollectionValue != value) {
123+
return new TableFunctionExpression((StreamingValue)translatedCollectionValue);
124+
}
125+
return this;
126+
}
127+
128+
@Nonnull
129+
@Override
130+
public Iterable<MatchInfo> subsumedBy(@Nonnull final RelationalExpression candidateExpression,
131+
@Nonnull final AliasMap bindingAliasMap,
132+
@Nonnull final IdentityBiMap<Quantifier, PartialMatch> partialMatchMap,
133+
@Nonnull final EvaluationContext evaluationContext) {
134+
if (!isCompatiblyAndCompletelyBound(bindingAliasMap, candidateExpression.getQuantifiers())) {
135+
return ImmutableList.of();
136+
}
137+
138+
return exactlySubsumedBy(candidateExpression, bindingAliasMap, partialMatchMap, TranslationMap.empty());
139+
}
140+
141+
@Nonnull
142+
@Override
143+
public Compensation compensate(@Nonnull final PartialMatch partialMatch,
144+
@Nonnull final Map<CorrelationIdentifier, ComparisonRange> boundParameterPrefixMap,
145+
@Nullable final PullUp pullUp,
146+
@Nonnull final CorrelationIdentifier nestingAlias) {
147+
// subsumedBy() is based on equality and this expression is always a leaf, thus we return empty here as
148+
// if there is a match, it's exact
149+
return Compensation.noCompensation();
150+
}
151+
152+
@Nonnull
153+
@Override
154+
public PlannerGraph rewriteInternalPlannerGraph(@Nonnull final List<? extends PlannerGraph> childGraphs) {
155+
return PlannerGraph.fromNodeAndChildGraphs(
156+
new PlannerGraph.LogicalOperatorNode(this,
157+
"TFunc",
158+
ImmutableList.of(toString()),
159+
ImmutableMap.of()),
160+
childGraphs);
161+
}
162+
163+
@Override
164+
public String toString() {
165+
return value.toString();
166+
}
167+
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import com.apple.foundationdb.record.query.plan.cascades.expressions.GroupByExpression;
2929
import com.apple.foundationdb.record.query.plan.cascades.expressions.InsertExpression;
3030
import com.apple.foundationdb.record.query.plan.cascades.expressions.RecursiveUnionExpression;
31+
import com.apple.foundationdb.record.query.plan.cascades.expressions.TableFunctionExpression;
3132
import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableInsertExpression;
3233
import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalDistinctExpression;
3334
import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalFilterExpression;
@@ -246,6 +247,11 @@ public static BindingMatcher<ExplodeExpression> explodeExpression() {
246247
return ofTypeOwning(ExplodeExpression.class, CollectionMatcher.empty());
247248
}
248249

250+
@Nonnull
251+
public static BindingMatcher<TableFunctionExpression> tableFunctionExpression() {
252+
return ofTypeOwning(TableFunctionExpression.class, CollectionMatcher.empty());
253+
}
254+
249255
@Nonnull
250256
public static BindingMatcher<GroupByExpression> groupByExpression(@Nonnull final BindingMatcher<? extends Quantifier> downstream) {
251257
return ofTypeOwning(GroupByExpression.class, only(downstream));

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/CompatibleTypeEvolutionPredicate.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase;
3838
import com.apple.foundationdb.record.query.plan.cascades.AliasMap;
3939
import com.apple.foundationdb.record.query.plan.cascades.BooleanWithConstraint;
40+
import com.apple.foundationdb.record.query.plan.cascades.values.FirstOrDefaultStreamingValue;
4041
import com.apple.foundationdb.record.query.plan.explain.ExplainTokens;
4142
import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence;
4243
import com.apple.foundationdb.record.query.plan.cascades.ValueEquivalence;
@@ -290,7 +291,7 @@ private static List<FieldAccessTrieNodeBuilder> computeFieldAccessForDerivation(
290291
return resultTrieBuilders.build();
291292
}
292293

293-
if (derivationValue instanceof FirstOrDefaultValue) {
294+
if (derivationValue instanceof FirstOrDefaultValue || derivationValue instanceof FirstOrDefaultStreamingValue) {
294295
Verify.verify(nestedResults.size() == 2);
295296
return nestedResults.get(0);
296297
}

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import com.apple.foundationdb.record.query.plan.cascades.expressions.GroupByExpression;
3737
import com.apple.foundationdb.record.query.plan.cascades.expressions.InsertExpression;
3838
import com.apple.foundationdb.record.query.plan.cascades.expressions.RecursiveUnionExpression;
39+
import com.apple.foundationdb.record.query.plan.cascades.expressions.TableFunctionExpression;
3940
import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableInsertExpression;
4041
import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalDistinctExpression;
4142
import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalFilterExpression;
@@ -75,6 +76,7 @@
7576
import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan;
7677
import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan;
7778
import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan;
79+
import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan;
7880
import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan;
7981
import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan;
8082
import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan;
@@ -248,6 +250,12 @@ public Cardinalities visitRecordQueryInsertPlan(@Nonnull final RecordQueryInsert
248250
return fromChild(insertPlan);
249251
}
250252

253+
@Nonnull
254+
@Override
255+
public Cardinalities visitRecordQueryTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) {
256+
return Cardinalities.unknownMaxCardinality();
257+
}
258+
251259
@Nonnull
252260
@Override
253261
public Cardinalities visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) {
@@ -559,6 +567,12 @@ public Cardinalities visitLogicalIntersectionExpression(@Nonnull final LogicalIn
559567
return intersectCardinalities(fromChildren(logicalIntersectionExpression));
560568
}
561569

570+
@Nonnull
571+
@Override
572+
public Cardinalities visitTableFunctionExpression(@Nonnull final TableFunctionExpression element) {
573+
return Cardinalities.unknownMaxCardinality();
574+
}
575+
562576
@Nonnull
563577
@Override
564578
public Cardinalities visitLogicalUniqueExpression(@Nonnull final LogicalUniqueExpression logicalUniqueExpression) {

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.apple.foundationdb.record.query.plan.cascades.Reference;
3030
import com.apple.foundationdb.record.query.plan.cascades.PlanProperty;
3131
import com.apple.foundationdb.record.query.plan.cascades.Quantifier;
32+
import com.apple.foundationdb.record.query.plan.cascades.values.FirstOrDefaultStreamingValue;
3233
import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap;
3334
import com.apple.foundationdb.record.query.plan.cascades.TreeLike;
3435
import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression;
@@ -60,6 +61,7 @@
6061
import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan;
6162
import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan;
6263
import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan;
64+
import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan;
6365
import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan;
6466
import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan;
6567
import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan;
@@ -336,6 +338,15 @@ public Derivations visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPl
336338
return new Derivations(resultValuesBuilder.build(), localValuesBuilder.build());
337339
}
338340

341+
@Nonnull
342+
@Override
343+
public Derivations visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan tableFunctionPlan) {
344+
final var streamingValue = tableFunctionPlan.getValue();
345+
final var elementType = streamingValue.getResultType();
346+
final var values = ImmutableList.<Value>of(new FirstOrDefaultStreamingValue(streamingValue, new ThrowsValue(elementType)));
347+
return new Derivations(values, values);
348+
}
349+
339350
@Nonnull
340351
@Override
341352
public Derivations visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) {

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan;
4848
import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan;
4949
import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan;
50+
import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan;
5051
import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan;
5152
import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan;
5253
import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan;
@@ -214,6 +215,12 @@ public Boolean visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPlan)
214215
return distinctRecordsFromSingleChild(insertPlan);
215216
}
216217

218+
@Nonnull
219+
@Override
220+
public Boolean visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) {
221+
return false;
222+
}
223+
217224
@Nonnull
218225
@Override
219226
public Boolean visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) {

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan;
6060
import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan;
6161
import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan;
62+
import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan;
6263
import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan;
6364
import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan;
6465
import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan;
@@ -302,6 +303,12 @@ public Ordering visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPlan)
302303
return Ordering.empty();
303304
}
304305

306+
@Nonnull
307+
@Override
308+
public Ordering visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) {
309+
return Ordering.empty();
310+
}
311+
305312
@Nonnull
306313
@Override
307314
public Ordering visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) {

0 commit comments

Comments
 (0)