Skip to content

Commit a75bf47

Browse files
authored
IGNITE-25818: Sql. Bump calcite version to 1.41 (#7249)
1 parent 423296c commit a75bf47

File tree

26 files changed

+394
-133
lines changed

26 files changed

+394
-133
lines changed

gradle/libs.versions.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ hamcrestOptional = "2.0.0"
5959
hamcrestPath = "1.0.1"
6060
hamcrestJson = "0.3"
6161
scalecube = "2.6.15"
62-
calcite = "1.40.0"
62+
calcite = "1.41.0"
6363
value = "2.12.0"
6464
janino = "3.1.12"
6565
jsonpath = "2.10.0"

modules/runner/src/main/java/org/apache/ignite/InitParametersBuilder.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import java.util.Collections;
2424
import java.util.List;
2525
import java.util.stream.Collectors;
26-
import javax.annotation.Nullable;
26+
import org.jetbrains.annotations.Nullable;
2727

2828
/** Builder of {@link InitParameters}. */
2929
public class InitParametersBuilder {

modules/sql-engine/src/integrationTest/sql/group1/aggregate/group/test_group_by.test

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -188,13 +188,17 @@ SELECT SUM(i) FROM integers GROUP BY GROUPING SETS (POWER(ROUND(ABS(SQRT(i*i)),
188188
3
189189
NULL
190190

191-
# expression with aliases can't be used in group by <columns> clause.
192-
statement error: Column 'K' not found in any table
191+
# expression with aliases can be used in group by <columns> clause.
192+
query II
193193
SELECT 1 AS k, SUM(i) FROM integers GROUP BY k+1 ORDER BY 2
194+
----
195+
1 6
194196

195-
# expression with aliases can't be used in group by <grouping sets> clause.
196-
statement error: Column 'K' not found in any table
197+
# expression with aliases can be used in group by <grouping sets> clause.
198+
query II
197199
SELECT 1 AS k, SUM(i) FROM integers GROUP BY GROUPING SETS ((k+1)) ORDER BY 2
200+
----
201+
1 6
198202

199203
# group by column refs should be recognized, even if one uses an explicit table specifier and the other does not
200204
query II

modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/exp/RexToLixTranslator.java

Lines changed: 53 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import org.apache.calcite.linq4j.tree.Primitive;
6161
import org.apache.calcite.linq4j.tree.Statement;
6262
import org.apache.calcite.rel.type.RelDataType;
63+
import org.apache.calcite.rel.type.RelDataTypeField;
6364
import org.apache.calcite.rex.RexBuilder;
6465
import org.apache.calcite.rex.RexCall;
6566
import org.apache.calcite.rex.RexCallBinding;
@@ -72,6 +73,7 @@
7273
import org.apache.calcite.rex.RexLiteral;
7374
import org.apache.calcite.rex.RexLocalRef;
7475
import org.apache.calcite.rex.RexNode;
76+
import org.apache.calcite.rex.RexNodeAndFieldIndex;
7577
import org.apache.calcite.rex.RexOver;
7678
import org.apache.calcite.rex.RexPatternFieldRef;
7779
import org.apache.calcite.rex.RexProgram;
@@ -82,6 +84,7 @@
8284
import org.apache.calcite.rex.RexVisitor;
8385
import org.apache.calcite.runtime.SpatialTypeFunctions;
8486
import org.apache.calcite.runtime.rtti.RuntimeTypeInformation;
87+
import org.apache.calcite.runtime.variant.VariantValue;
8588
import org.apache.calcite.schema.FunctionContext;
8689
import org.apache.calcite.sql.SqlIntervalQualifier;
8790
import org.apache.calcite.sql.SqlOperator;
@@ -354,53 +357,54 @@ private Expression getConvertExpression(
354357
final Supplier<Expression> defaultExpression = () ->
355358
ConverterUtils.convert(operand, targetType);
356359

357-
// if (sourceType.getSqlTypeName() == SqlTypeName.VARIANT) {
358-
// // Converting VARIANT to VARIANT uses the default conversion
359-
// if (targetType.getSqlTypeName() == SqlTypeName.VARIANT) {
360-
// return defaultExpression.get();
361-
// }
362-
// // Converting a VARIANT to any other type calls the Variant.cast method
363-
// // First cast operand to a VariantValue (it may be an Object)
364-
// Expression operandCast = Expressions.convert_(operand, VariantValue.class);
365-
// Expression cast =
366-
// Expressions.call(operandCast, BuiltInMethod.VARIANT_CAST.method,
367-
// RuntimeTypeInformation.createExpression(targetType));
368-
// // The cast returns an Object, so we need a convert to the expected Java type
369-
// RelDataType nullableTarget = typeFactory.createTypeWithNullability(targetType, true);
370-
// return Expressions.convert_(cast, typeFactory.getJavaClass(nullableTarget));
371-
// }
372-
//
373-
// if (targetType.getSqlTypeName() == SqlTypeName.ROW) {
374-
// assert sourceType.getSqlTypeName() == SqlTypeName.ROW;
375-
// List<RelDataTypeField> targetTypes = targetType.getFieldList();
376-
// List<RelDataTypeField> sourceTypes = sourceType.getFieldList();
377-
// assert targetTypes.size() == sourceTypes.size();
378-
// List<Expression> fields = new ArrayList<>();
379-
// for (int i = 0; i < targetTypes.size(); i++) {
380-
// RelDataTypeField targetField = targetTypes.get(i);
381-
// RelDataTypeField sourceField = sourceTypes.get(i);
382-
// Expression field = Expressions.arrayIndex(operand, Expressions.constant(i));
383-
// // In the generated Java code 'field' is an Object,
384-
// // we need to also cast it to the correct type to enable correct method dispatch in Java.
385-
// // We force the type to be nullable; this way, instead of (int) we get (Integer).
386-
// // Casting an object ot an int is not legal.
387-
// RelDataType nullableSourceFieldType =
388-
// typeFactory.createTypeWithNullability(sourceField.getType(), true);
389-
// Type javaType = typeFactory.getJavaClass(nullableSourceFieldType);
390-
// if (!javaType.getTypeName().equals("java.lang.Void")
391-
// && !nullableSourceFieldType.isStruct()) {
392-
// // Cannot cast to Void - this is the type of NULL literals.
393-
// field = Expressions.convert_(field, javaType);
394-
// }
395-
// Expression convert =
396-
// getConvertExpression(sourceField.getType(), targetField.getType(), field, format);
397-
// fields.add(convert);
398-
// }
399-
// return Expressions.call(BuiltInMethod.ARRAY.method, fields);
400-
// }
360+
if (sourceType.getSqlTypeName() == SqlTypeName.VARIANT) {
361+
// Converting VARIANT to VARIANT uses the default conversion
362+
if (targetType.getSqlTypeName() == SqlTypeName.VARIANT) {
363+
return defaultExpression.get();
364+
}
365+
// Converting a VARIANT to any other type calls the Variant.cast method
366+
// First cast operand to a VariantValue (it may be an Object)
367+
Expression operandCast = Expressions.convert_(operand, VariantValue.class);
368+
Expression cast =
369+
Expressions.call(operandCast, BuiltInMethod.VARIANT_CAST.method,
370+
RuntimeTypeInformation.createExpression(targetType));
371+
// The cast returns an Object, so we need a convert to the expected Java type
372+
RelDataType nullableTarget = typeFactory.createTypeWithNullability(targetType, true);
373+
return Expressions.convert_(cast, typeFactory.getJavaClass(nullableTarget));
374+
}
375+
376+
if (targetType.getSqlTypeName() == SqlTypeName.ROW) {
377+
assert sourceType.getSqlTypeName() == SqlTypeName.ROW;
378+
List<RelDataTypeField> targetTypes = targetType.getFieldList();
379+
List<RelDataTypeField> sourceTypes = sourceType.getFieldList();
380+
assert targetTypes.size() == sourceTypes.size();
381+
List<Expression> fields = new ArrayList<>();
382+
for (int i = 0; i < targetTypes.size(); i++) {
383+
RelDataTypeField targetField = targetTypes.get(i);
384+
RelDataTypeField sourceField = sourceTypes.get(i);
385+
Expression field = Expressions.arrayIndex(operand, Expressions.constant(i));
386+
// In the generated Java code 'field' is an Object,
387+
// we need to also cast it to the correct type to enable correct method dispatch in Java.
388+
// We force the type to be nullable; this way, instead of (int) we get (Integer).
389+
// Casting an object ot an int is not legal.
390+
RelDataType nullableSourceFieldType =
391+
typeFactory.createTypeWithNullability(sourceField.getType(), true);
392+
Type javaType = typeFactory.getJavaClass(nullableSourceFieldType);
393+
if (!javaType.getTypeName().equals("java.lang.Void")
394+
&& !nullableSourceFieldType.isStruct()) {
395+
// Cannot cast to Void - this is the type of NULL literals.
396+
field = Expressions.convert_(field, javaType);
397+
}
398+
Expression convert =
399+
getConvertExpression(sourceField.getType(), targetField.getType(), field, format);
400+
fields.add(convert);
401+
}
402+
return Expressions.call(BuiltInMethod.ARRAY.method, fields);
403+
}
401404

402405
switch (targetType.getSqlTypeName()) {
403406
case ARRAY:
407+
case MULTISET:
404408
final RelDataType sourceDataType = sourceType.getComponentType();
405409
final RelDataType targetDataType = targetType.getComponentType();
406410
assert sourceDataType != null;
@@ -1908,6 +1912,11 @@ private Result toInnerStorageType(Result result, Type storageType) {
19081912
return new Result(isNullVariable, valueVariable);
19091913
}
19101914

1915+
@Override
1916+
public Result visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
1917+
throw new RuntimeException("cannot translate expression " + nodeAndFieldIndex);
1918+
}
1919+
19111920
Expression checkNull(Expression expr) {
19121921
if (Primitive.flavor(expr.getType())
19131922
== Primitive.Flavor.PRIMITIVE) {

modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgniteSqlToRelConvertor.java

Lines changed: 114 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,17 @@
1919

2020
import static java.util.Objects.requireNonNull;
2121

22+
import java.lang.invoke.MethodHandle;
23+
import java.lang.invoke.MethodHandles;
24+
import java.lang.invoke.MethodHandles.Lookup;
25+
import java.lang.invoke.MethodType;
2226
import java.util.ArrayDeque;
2327
import java.util.ArrayList;
2428
import java.util.Deque;
2529
import java.util.List;
2630
import org.apache.calcite.plan.RelOptCluster;
2731
import org.apache.calcite.plan.RelOptTable;
32+
import org.apache.calcite.plan.RelOptUtil;
2833
import org.apache.calcite.prepare.Prepare;
2934
import org.apache.calcite.rel.RelNode;
3035
import org.apache.calcite.rel.RelRoot;
@@ -43,22 +48,50 @@
4348
import org.apache.calcite.sql.SqlKind;
4449
import org.apache.calcite.sql.SqlMerge;
4550
import org.apache.calcite.sql.SqlNode;
51+
import org.apache.calcite.sql.SqlSelect;
4652
import org.apache.calcite.sql.SqlUpdate;
47-
import org.apache.calcite.sql.util.SqlShuttle;
4853
import org.apache.calcite.sql.validate.SqlValidator;
4954
import org.apache.calcite.sql.validate.SqlValidatorScope;
5055
import org.apache.calcite.sql.validate.SqlValidatorUtil;
5156
import org.apache.calcite.sql2rel.InitializerContext;
5257
import org.apache.calcite.sql2rel.SqlRexConvertletTable;
5358
import org.apache.calcite.sql2rel.SqlToRelConverter;
5459
import org.apache.calcite.tools.RelBuilder;
55-
import org.apache.calcite.util.ControlFlowException;
5660
import org.apache.calcite.util.Pair;
61+
import org.apache.calcite.util.Util;
5762
import org.apache.ignite.internal.sql.engine.schema.IgniteDataSource;
5863
import org.jetbrains.annotations.Nullable;
5964

6065
/** Converts a SQL parse tree into a relational algebra operators. */
6166
public class IgniteSqlToRelConvertor extends SqlToRelConverter implements InitializerContext {
67+
68+
private static final MethodHandle REPLACE_SUB_QUERIES;
69+
70+
private static final Throwable INIT_ERROR;
71+
72+
static {
73+
// TODO https://issues.apache.org/jira/browse/IGNITE-27398 Remove this workaround
74+
MethodHandle handle;
75+
Throwable err;
76+
try {
77+
Lookup lookup = MethodHandles.privateLookupIn(SqlToRelConverter.class, MethodHandles.lookup());
78+
Class<?> bbClass = lookup.findClass("org.apache.calcite.sql2rel.SqlToRelConverter$Blackboard");
79+
Class<?> logicClass = lookup.findClass("org.apache.calcite.plan.RelOptUtil$Logic");
80+
Class<?> sqlNodeClass = SqlNode.class;
81+
MethodType tpe = MethodType.methodType(void.class, bbClass, sqlNodeClass, logicClass);
82+
83+
handle = lookup.findVirtual(SqlToRelConverter.class, "replaceSubQueries", tpe);
84+
err = null;
85+
} catch (Throwable e) {
86+
// Postpone error reporting to have a better chance of logging an error.
87+
err = e;
88+
handle = null;
89+
}
90+
91+
INIT_ERROR = err;
92+
REPLACE_SUB_QUERIES = handle;
93+
}
94+
6295
private final Deque<SqlCall> datasetStack = new ArrayDeque<>();
6396

6497
private RelBuilder relBuilder;
@@ -73,12 +106,19 @@ public class IgniteSqlToRelConvertor extends SqlToRelConverter implements Initia
73106
super(viewExpander, validator, catalogReader, cluster, convertletTable, cfg);
74107

75108
relBuilder = config.getRelBuilderFactory().create(cluster, null);
109+
110+
if (INIT_ERROR != null) {
111+
throw new IllegalStateException("Failed to initialize " + IgniteSqlToRelConvertor.class.getName(), INIT_ERROR);
112+
}
76113
}
77114

78115
/** {@inheritDoc} */
79116
@Override protected RelRoot convertQueryRecursive(SqlNode qry, boolean top, @Nullable RelDataType targetRowType) {
80117
if (qry.getKind() == SqlKind.MERGE) {
81118
return RelRoot.of(convertMerge((SqlMerge) qry), qry.getKind());
119+
} else if (qry.getKind() == SqlKind.UPDATE) {
120+
// TODO https://issues.apache.org/jira/browse/IGNITE-27398 Remove this workaround and use calcite's method.
121+
return RelRoot.of(convertUpdateFixed((SqlUpdate) qry), qry.getKind());
82122
} else {
83123
return super.convertQueryRecursive(qry, top, targetRowType);
84124
}
@@ -99,39 +139,30 @@ public SqlNode validateExpression(RelDataType rowType, SqlNode expr) {
99139
throw new UnsupportedOperationException("Not implemented yet");
100140
}
101141

102-
private static class DefaultChecker extends SqlShuttle {
103-
private boolean hasDefaults(SqlCall call) {
104-
try {
105-
call.accept(this);
106-
return false;
107-
} catch (ControlFlowException e) {
108-
return true;
109-
}
110-
}
111-
112-
@Override public @Nullable SqlNode visit(SqlCall call) {
113-
if (call.getKind() == SqlKind.DEFAULT) {
114-
throw new ControlFlowException();
115-
}
116-
117-
return super.visit(call);
118-
}
119-
}
120-
121142
@Override public RelNode convertValues(SqlCall values, RelDataType targetRowType) {
122-
DefaultChecker checker = new DefaultChecker();
123-
124-
boolean hasDefaults = checker.hasDefaults(values);
125-
126-
if (hasDefaults) {
143+
// TODO https://issues.apache.org/jira/browse/IGNITE-27396
144+
// FIX: Original convertValuesImpl adds additional type casts that are not correct
145+
// and break NOT NULL constraints.
146+
//
147+
// See these lines in convertValuesImpl:
148+
// if (!(def instanceof RexDynamicParam) && !def.getType().equals(fieldType)) {
149+
// def = rexBuilder.makeCast(operand.getParserPosition(), fieldType, def);
150+
// }
151+
// exps.add(def, SqlValidatorUtil.alias(operand, i));
152+
//
153+
// Example: INSERT INTO t1 VALUES(1, (SELECT NULL))
154+
//
155+
// if fieldType is NOT NULLABLE INT and def's type is NULLABLE INT then
156+
// resulting expression is wrapped into CAST(NULLABLE INT AS NOT NULLABLE INT)
157+
// but that cast expression always results in 0 (INT) thus breaking a NOT NULL constraint.
158+
if (datasetStack.peek() instanceof SqlInsert) {
127159
SqlValidatorScope scope = validator.getOverScope(values);
128160
assert scope != null;
129161
Blackboard bb = createBlackboard(scope, null, false);
130162

131163
convertValuesImplEx(bb, values, targetRowType);
132164
return bb.root();
133165
} else {
134-
// a bit lightweight than default processing one.
135166
return super.convertValues(values, targetRowType);
136167
}
137168
}
@@ -345,4 +376,60 @@ private List<RexNode> repairProject(LogicalJoin join, List<RexNode> actual) {
345376
public RelOptTable getTargetTable(SqlNode call) {
346377
return super.getTargetTable(call);
347378
}
379+
380+
// TODO https://issues.apache.org/jira/browse/IGNITE-27398 Remove this workaround
381+
// This method is a copy of SqlToRelConverter convertUpdate.
382+
private RelNode convertUpdateFixed(SqlUpdate call) {
383+
final SqlSelect sourceSelect =
384+
requireNonNull(call.getSourceSelect(),
385+
() -> "sourceSelect for " + call);
386+
final SqlValidatorScope scope = validator.getWhereScope(sourceSelect);
387+
Blackboard bb = createBlackboard(scope, null, false);
388+
389+
RelOptTable targetTable = getTargetTable(call);
390+
391+
// convert update column list from SqlIdentifier to String
392+
final List<String> targetColumnNameList = new ArrayList<>();
393+
final RelDataType targetRowType = targetTable.getRowType();
394+
for (SqlNode node : call.getTargetColumnList()) {
395+
SqlIdentifier id = (SqlIdentifier) node;
396+
RelDataTypeField field =
397+
SqlValidatorUtil.getTargetField(
398+
targetRowType, typeFactory, id, catalogReader, targetTable);
399+
if (field == null) {
400+
throw new AssertionError("column " + id + " not found");
401+
}
402+
targetColumnNameList.add(field.getName());
403+
}
404+
405+
// A call to replaceSubQueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
406+
try {
407+
REPLACE_SUB_QUERIES.invoke(this, bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN);
408+
} catch (Throwable e) {
409+
throw new AssertionError("Failed to replace subqueries", e);
410+
}
411+
412+
// FIX: The condition below is incorrect, it ignores virtual columns that present is the target table.
413+
// sourceSelect` should contain target columns values plus source expressions
414+
//
415+
// if (sourceSelect.getSelectList().size()
416+
// != targetTable.getRowType().getFieldCount() + call.getSourceExpressionList().size()) {
417+
// throw new AssertionError(
418+
// "Unexpected select list size. Select list should contain both target table columns and "
419+
// + "set expressions");
420+
// }
421+
422+
RelNode sourceRel = convertSelect(sourceSelect, false);
423+
bb.setRoot(sourceRel, false);
424+
425+
// sourceRel already contains all source expressions. Only create references to those fields.
426+
List<RexNode> rexExpressionList =
427+
Util.transform(
428+
Util.last(sourceRel.getRowType().getFieldList(), targetColumnNameList.size()),
429+
expressionField -> new RexInputRef(expressionField.getIndex(),
430+
expressionField.getType()));
431+
432+
return LogicalTableModify.create(targetTable, catalogReader, sourceRel,
433+
LogicalTableModify.Operation.UPDATE, targetColumnNameList, rexExpressionList, false);
434+
}
348435
}

0 commit comments

Comments
 (0)