Skip to content

Commit b9e9e9d

Browse files
xiedeyantuxiedeyantu
committed
[CALCITE-7342] Quidem test support for TopDownGeneralDecorrelator
1 parent 02c176a commit b9e9e9d

File tree

16 files changed

+294
-85
lines changed

16 files changed

+294
-85
lines changed

‎core/src/main/java/org/apache/calcite/config/CalciteConnectionConfig.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,9 @@ public interface CalciteConnectionConfig extends ConnectionConfig {
110110
boolean lenientOperatorLookup();
111111
/** Returns the value of {@link CalciteConnectionProperty#TOPDOWN_OPT}. */
112112
boolean topDownOpt();
113+
/** Returns the value of
114+
* {@link CalciteConnectionProperty#TOPDOWN_GENERAL_DECORRELATION_ENABLED}. */
115+
boolean topDownGeneralDecorrelationEnabled();
113116

114117
/** Returns the value of {@link CalciteConnectionProperty#META_TABLE_FACTORY},
115118
* or a default meta table factory if not set. If

‎core/src/main/java/org/apache/calcite/config/CalciteConnectionConfigImpl.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,11 @@ public boolean isSet(CalciteConnectionProperty property) {
215215
.getBoolean();
216216
}
217217

218+
@Override public boolean topDownGeneralDecorrelationEnabled() {
219+
return CalciteConnectionProperty.TOPDOWN_GENERAL_DECORRELATION_ENABLED.wrap(properties)
220+
.getBoolean();
221+
}
222+
218223
@Override public <T> @PolyNull T metaTableFactory(
219224
Class<T> metaTableFactoryClass,
220225
@PolyNull T defaultMetaTableFactory) {

‎core/src/main/java/org/apache/calcite/config/CalciteConnectionProperty.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,9 @@ public enum CalciteConnectionProperty implements ConnectionProperty {
154154
* If true (the default), Calcite de-correlates the plan. */
155155
FORCE_DECORRELATE("forceDecorrelate", Type.BOOLEAN, true, false),
156156

157+
TOPDOWN_GENERAL_DECORRELATION_ENABLED("topDownGeneralDecorrelationEnabled",
158+
Type.BOOLEAN, false, false),
159+
157160
/** Type system. The name of a class that implements
158161
* {@link org.apache.calcite.rel.type.RelDataTypeSystem} and has a public
159162
* default constructor or an {@code INSTANCE} constant. */

‎core/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,10 @@
107107
import org.apache.calcite.sql2rel.SqlRexConvertletTable;
108108
import org.apache.calcite.sql2rel.SqlToRelConverter;
109109
import org.apache.calcite.sql2rel.StandardConvertletTable;
110+
import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator;
110111
import org.apache.calcite.tools.FrameworkConfig;
111112
import org.apache.calcite.tools.Frameworks;
113+
import org.apache.calcite.tools.RelBuilder;
112114
import org.apache.calcite.util.ImmutableIntList;
113115
import org.apache.calcite.util.Pair;
114116
import org.apache.calcite.util.Util;
@@ -1091,6 +1093,9 @@ private PreparedResult prepare_(Supplier<RelNode> fn,
10911093
SqlValidator validator,
10921094
CatalogReader catalogReader,
10931095
SqlToRelConverter.Config config) {
1096+
config =
1097+
config.withTopDownGeneralDecorrelationEnabled(
1098+
context.config().topDownGeneralDecorrelationEnabled());
10941099
return new SqlToRelConverter(this, validator, catalogReader, cluster,
10951100
convertletTable, config);
10961101
}
@@ -1107,6 +1112,11 @@ private PreparedResult prepare_(Supplier<RelNode> fn,
11071112

11081113
@Override protected RelNode decorrelate(SqlToRelConverter sqlToRelConverter,
11091114
SqlNode query, RelNode rootRel) {
1115+
if (context.config().topDownGeneralDecorrelationEnabled()) {
1116+
final RelBuilder relBuilder =
1117+
sqlToRelConverter.config().getRelBuilderFactory().create(rootRel.getCluster(), null);
1118+
return TopDownGeneralDecorrelator.decorrelateQuery(rootRel, relBuilder);
1119+
}
11101120
return sqlToRelConverter.decorrelate(query, rootRel);
11111121
}
11121122

‎core/src/main/java/org/apache/calcite/prepare/PlannerImpl.java‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.apache.calcite.sql2rel.RelDecorrelator;
5252
import org.apache.calcite.sql2rel.SqlRexConvertletTable;
5353
import org.apache.calcite.sql2rel.SqlToRelConverter;
54+
import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator;
5455
import org.apache.calcite.tools.FrameworkConfig;
5556
import org.apache.calcite.tools.Planner;
5657
import org.apache.calcite.tools.Program;
@@ -262,8 +263,10 @@ private void ready() {
262263
final RelOptCluster cluster =
263264
RelOptCluster.create(requireNonNull(planner, "planner"),
264265
rexBuilder);
265-
final SqlToRelConverter.Config config =
266-
sqlToRelConverterConfig.withTrimUnusedFields(false);
266+
final SqlToRelConverter.Config config = sqlToRelConverterConfig
267+
.withTrimUnusedFields(false)
268+
.withTopDownGeneralDecorrelationEnabled(
269+
connectionConfig.topDownGeneralDecorrelationEnabled());
267270
final SqlToRelConverter sqlToRelConverter =
268271
new SqlToRelConverter(this, validator,
269272
createCatalogReader(), cluster, convertletTable, config);
@@ -272,8 +275,9 @@ private void ready() {
272275
root = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true));
273276
final RelBuilder relBuilder =
274277
config.getRelBuilderFactory().create(cluster, null);
275-
root =
276-
root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder));
278+
root = config.isTopDownGeneralDecorrelationEnabled()
279+
? root.withRel(TopDownGeneralDecorrelator.decorrelateQuery(root.rel, relBuilder))
280+
: root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder));
277281
state = State.STATE_5_CONVERTED;
278282
return root;
279283
}
@@ -314,20 +318,22 @@ public class ViewExpanderImpl implements ViewExpander {
314318

315319
final RexBuilder rexBuilder = createRexBuilder();
316320
final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder);
317-
final SqlToRelConverter.Config config =
318-
sqlToRelConverterConfig.withTrimUnusedFields(false);
321+
final SqlToRelConverter.Config config = sqlToRelConverterConfig
322+
.withTrimUnusedFields(false)
323+
.withTopDownGeneralDecorrelationEnabled(
324+
connectionConfig.topDownGeneralDecorrelationEnabled());
319325
final SqlToRelConverter sqlToRelConverter =
320326
new SqlToRelConverter(this, validator,
321327
catalogReader, cluster, convertletTable, config);
322328

323-
final RelRoot root =
329+
RelRoot root =
324330
sqlToRelConverter.convertQuery(sqlNode, true, false);
325-
final RelRoot root2 =
326-
root.withRel(sqlToRelConverter.flattenTypes(root.rel, true));
331+
root = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true));
327332
final RelBuilder relBuilder =
328333
config.getRelBuilderFactory().create(cluster, null);
329-
return root2.withRel(
330-
RelDecorrelator.decorrelateQuery(root.rel, relBuilder));
334+
return config.isTopDownGeneralDecorrelationEnabled()
335+
? root.withRel(TopDownGeneralDecorrelator.decorrelateQuery(root.rel, relBuilder))
336+
: root.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder));
331337
}
332338

333339
// CalciteCatalogReader is stateless; no need to store one

‎core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -175,11 +175,10 @@ protected FilterProjectTransposeRule(
175175
final RelNode input = project.getInput();
176176
final RelTraitSet traitSet = filter.getTraitSet()
177177
.replaceIfs(RelCollationTraitDef.INSTANCE,
178-
() -> Collections.singletonList(
179-
input.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE)))
178+
() -> input.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE))
180179
.replaceIfs(RelDistributionTraitDef.INSTANCE,
181180
() -> Collections.singletonList(
182-
input.getTraitSet().getTrait(RelDistributionTraitDef.INSTANCE)));
181+
input.getTraitSet().getTrait(RelDistributionTraitDef.INSTANCE)));
183182
newCondition = RexUtil.removeNullabilityCast(relBuilder.getTypeFactory(), newCondition);
184183
newFilterRel = filter.copy(traitSet, input, newCondition);
185184
} else {

‎core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3966,6 +3966,9 @@ protected boolean enableDecorrelation() {
39663966
}
39673967

39683968
protected RelNode decorrelateQuery(RelNode rootRel) {
3969+
if (config.isTopDownGeneralDecorrelationEnabled()) {
3970+
return TopDownGeneralDecorrelator.decorrelateQuery(rootRel, relBuilder);
3971+
}
39693972
return RelDecorrelator.decorrelateQuery(rootRel, relBuilder);
39703973
}
39713974

@@ -6496,6 +6499,14 @@ public interface Config {
64966499
/** Sets {@link #isDecorrelationEnabled()}. */
64976500
Config withDecorrelationEnabled(boolean decorrelationEnabled);
64986501

6502+
/** Returns whether to use the top-down general decorrelator. */
6503+
@Value.Default default boolean isTopDownGeneralDecorrelationEnabled() {
6504+
return false;
6505+
}
6506+
6507+
/** Sets {@link #isTopDownGeneralDecorrelationEnabled()}. */
6508+
Config withTopDownGeneralDecorrelationEnabled(boolean topDownGeneralDecorrelationEnabled);
6509+
64996510
/** Returns the {@code trimUnusedFields} option. Controls whether to trim
65006511
* unused fields as part of the conversion process. */
65016512
@Value.Default default boolean isTrimUnusedFields() {

‎core/src/main/java/org/apache/calcite/tools/Programs.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import org.apache.calcite.sql2rel.RelDecorrelator;
4747
import org.apache.calcite.sql2rel.RelFieldTrimmer;
4848
import org.apache.calcite.sql2rel.SqlToRelConverter;
49+
import org.apache.calcite.sql2rel.TopDownGeneralDecorrelator;
4950
import org.apache.calcite.util.Util;
5051

5152
import com.google.common.collect.ImmutableList;
@@ -259,7 +260,26 @@ public static Program subQuery(RelMetadataProvider metadataProvider) {
259260
CoreRules.PROJECT_SUB_QUERY_TO_CORRELATE,
260261
CoreRules.JOIN_SUB_QUERY_TO_CORRELATE,
261262
CoreRules.PROJECT_OVER_SUM_TO_SUM0_RULE));
262-
return of(builder.build(), true, metadataProvider);
263+
final Program oldProgram = of(builder.build(), true, metadataProvider);
264+
265+
final HepProgramBuilder newBuilder = HepProgram.builder();
266+
newBuilder.addRuleCollection(
267+
ImmutableList.of(CoreRules.FILTER_SUB_QUERY_TO_MARK_CORRELATE,
268+
CoreRules.PROJECT_SUB_QUERY_TO_MARK_CORRELATE,
269+
CoreRules.JOIN_SUB_QUERY_TO_CORRELATE,
270+
CoreRules.PROJECT_OVER_SUM_TO_SUM0_RULE));
271+
final Program newProgram = of(newBuilder.build(), true, metadataProvider);
272+
273+
return (planner, rel, requiredOutputTraits, materializations, lattices) -> {
274+
final CalciteConnectionConfig config =
275+
planner.getContext().maybeUnwrap(CalciteConnectionConfig.class)
276+
.orElse(CalciteConnectionConfig.DEFAULT);
277+
final Program program = config.topDownGeneralDecorrelationEnabled()
278+
? newProgram
279+
: oldProgram;
280+
return program.run(planner, rel, requiredOutputTraits, materializations,
281+
lattices);
282+
};
263283
}
264284

265285
public static Program measure(RelMetadataProvider metadataProvider) {
@@ -430,6 +450,9 @@ private static class DecorrelateProgram implements Program {
430450
if (config.forceDecorrelate()) {
431451
final RelBuilder relBuilder =
432452
RelFactories.LOGICAL_BUILDER.create(rel.getCluster(), null);
453+
if (config.topDownGeneralDecorrelationEnabled()) {
454+
return TopDownGeneralDecorrelator.decorrelateQuery(rel, relBuilder);
455+
}
433456
return RelDecorrelator.decorrelateQuery(rel, relBuilder);
434457
}
435458
return rel;

‎core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java‎

Lines changed: 31 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
/**
4242
* Test that runs every Quidem file in the "core" module as a test.
4343
*/
44-
class CoreQuidemTest extends QuidemTest {
44+
public class CoreQuidemTest extends QuidemTest {
4545
/** Runs a test from the command line.
4646
*
4747
* <p>For example:
@@ -57,6 +57,12 @@ public static void main(String[] args) throws Exception {
5757

5858
/** For {@link QuidemTest#test(String)} parameters. */
5959
@Override public Collection<String> getPath() {
60+
return data();
61+
}
62+
63+
/** Returns the list of Quidem files to run.
64+
* Subclasses can override this method to gradually add files. */
65+
protected Collection<String> data() {
6066
// Start with a test file we know exists, then find the directory and list
6167
// its files.
6268
final String first = "sql/agg.iq";
@@ -68,116 +74,116 @@ public static void main(String[] args) throws Exception {
6874
@Override public Connection connect(String name, boolean reference) throws Exception {
6975
switch (name) {
7076
case "blank":
71-
return CalciteAssert.that()
77+
return customize(CalciteAssert.that()
7278
.with(CalciteConnectionProperty.PARSER_FACTORY,
7379
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
74-
.with(CalciteAssert.SchemaSpec.BLANK)
80+
.with(CalciteAssert.SchemaSpec.BLANK))
7581
.connect();
7682
case "scott":
77-
return CalciteAssert.that()
83+
return customize(CalciteAssert.that()
7884
.with(CalciteConnectionProperty.PARSER_FACTORY,
7985
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
8086
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
81-
.with(CalciteAssert.Config.SCOTT)
87+
.with(CalciteAssert.Config.SCOTT))
8288
.connect();
8389
case "scott-spark":
8490
discard(CustomTypeSystems.SPARK_TYPE_SYSTEM);
85-
return CalciteAssert.that()
91+
return customize(CalciteAssert.that()
8692
.with(CalciteConnectionProperty.PARSER_FACTORY,
8793
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
8894
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
8995
.with(CalciteConnectionProperty.TYPE_SYSTEM,
9096
CustomTypeSystems.class.getName() + "#SPARK_TYPE_SYSTEM")
91-
.with(CalciteAssert.Config.SCOTT)
97+
.with(CalciteAssert.Config.SCOTT))
9298
.connect();
9399
case "scott-checked-rounding-half-up":
94100
discard(CustomTypeSystems.ROUNDING_MODE_HALF_UP);
95-
return CalciteAssert.that()
101+
return customize(CalciteAssert.that()
96102
.with(CalciteConnectionProperty.PARSER_FACTORY,
97103
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
98104
// Use bigquery conformance, which forces checked arithmetic
99105
.with(CalciteConnectionProperty.CONFORMANCE, SqlConformanceEnum.BIG_QUERY)
100106
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
101107
.with(CalciteConnectionProperty.TYPE_SYSTEM,
102108
CustomTypeSystems.class.getName() + "#ROUNDING_MODE_HALF_UP")
103-
.with(CalciteAssert.Config.SCOTT)
109+
.with(CalciteAssert.Config.SCOTT))
104110
.connect();
105111
case "scott-negative-scale":
106112
discard(CustomTypeSystems.NEGATIVE_SCALE);
107-
return CalciteAssert.that()
113+
return customize(CalciteAssert.that()
108114
.with(CalciteConnectionProperty.PARSER_FACTORY,
109115
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
110116
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
111117
.with(CalciteConnectionProperty.TYPE_SYSTEM,
112118
CustomTypeSystems.class.getName() + "#NEGATIVE_SCALE")
113-
.with(CalciteAssert.Config.SCOTT)
119+
.with(CalciteAssert.Config.SCOTT))
114120
.connect();
115121
case "scott-negative-scale-rounding-half-up":
116122
discard(CustomTypeSystems.NEGATIVE_SCALE_ROUNDING_MODE_HALF_UP);
117-
return CalciteAssert.that()
123+
return customize(CalciteAssert.that()
118124
.with(CalciteConnectionProperty.PARSER_FACTORY,
119125
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
120126
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
121127
.with(CalciteConnectionProperty.TYPE_SYSTEM,
122128
CustomTypeSystems.class.getName()
123129
+ "#NEGATIVE_SCALE_ROUNDING_MODE_HALF_UP")
124-
.with(CalciteAssert.Config.SCOTT)
130+
.with(CalciteAssert.Config.SCOTT))
125131
.connect();
126132
case "scott-lenient":
127133
// Same as "scott", but uses LENIENT conformance.
128134
// TODO: add a way to change conformance without defining a new
129135
// connection
130-
return CalciteAssert.that()
136+
return customize(CalciteAssert.that()
131137
.with(CalciteConnectionProperty.PARSER_FACTORY,
132138
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
133139
.with(CalciteConnectionProperty.CONFORMANCE,
134140
SqlConformanceEnum.LENIENT)
135-
.with(CalciteAssert.Config.SCOTT)
141+
.with(CalciteAssert.Config.SCOTT))
136142
.connect();
137143
case "scott-babel":
138144
// Same as "scott", but uses BABEL conformance.
139145
// connection
140-
return CalciteAssert.that()
146+
return customize(CalciteAssert.that()
141147
.with(CalciteConnectionProperty.PARSER_FACTORY,
142148
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
143149
.with(CalciteConnectionProperty.CONFORMANCE,
144150
SqlConformanceEnum.BABEL)
145-
.with(CalciteAssert.Config.SCOTT)
151+
.with(CalciteAssert.Config.SCOTT))
146152
.connect();
147153
case "scott-mysql":
148154
// Same as "scott", but uses MySQL conformance.
149-
return CalciteAssert.that()
155+
return customize(CalciteAssert.that()
150156
.with(CalciteConnectionProperty.PARSER_FACTORY,
151157
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
152158
.with(CalciteConnectionProperty.CONFORMANCE,
153159
SqlConformanceEnum.MYSQL_5)
154-
.with(CalciteAssert.Config.SCOTT)
160+
.with(CalciteAssert.Config.SCOTT))
155161
.connect();
156162
case "scott-oracle":
157163
// Same as "scott", but uses Oracle conformance.
158-
return CalciteAssert.that()
164+
return customize(CalciteAssert.that()
159165
.with(CalciteConnectionProperty.PARSER_FACTORY,
160166
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
161167
.with(CalciteConnectionProperty.CONFORMANCE,
162168
SqlConformanceEnum.ORACLE_10)
163-
.with(CalciteAssert.Config.SCOTT)
169+
.with(CalciteAssert.Config.SCOTT))
164170
.connect();
165171
case "scott-mssql":
166172
// Same as "scott", but uses SQL_SERVER_2008 conformance.
167-
return CalciteAssert.that()
173+
return customize(CalciteAssert.that()
168174
.with(CalciteConnectionProperty.PARSER_FACTORY,
169175
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
170176
.with(CalciteConnectionProperty.CONFORMANCE,
171177
SqlConformanceEnum.SQL_SERVER_2008)
172-
.with(CalciteAssert.Config.SCOTT)
178+
.with(CalciteAssert.Config.SCOTT))
173179
.connect();
174180
case "steelwheels":
175-
return CalciteAssert.that()
181+
return customize(CalciteAssert.that()
176182
.with(CalciteConnectionProperty.PARSER_FACTORY,
177183
ExtensionDdlExecutor.class.getName() + "#PARSER_FACTORY")
178184
.with(CalciteConnectionProperty.FUN, SqlLibrary.CALCITE.fun)
179185
.with(CalciteAssert.SchemaSpec.STEELWHEELS)
180-
.with(Lex.BIG_QUERY)
186+
.with(Lex.BIG_QUERY))
181187
.connect();
182188
default:
183189
return super.connect(name, reference);

0 commit comments

Comments
 (0)