Skip to content

Commit 1be4be0

Browse files
guojn1githubgxll
guojn1
authored and
githubgxll
committed
[fix][dingo-executor] Improve the performance of adding multiple column
1 parent 504311c commit 1be4be0

File tree

14 files changed

+475
-228
lines changed

14 files changed

+475
-228
lines changed

‎dingo-calcite/src/main/java/io/dingodb/calcite/DingoDdlExecutor.java‎

Lines changed: 106 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,6 @@
131131
import io.dingodb.common.util.Parameters;
132132
import io.dingodb.common.util.Utils;
133133
import io.dingodb.expr.common.timezone.core.DateTimeType;
134-
import io.dingodb.expr.common.type.AnyType;
135134
import io.dingodb.meta.DdlService;
136135
import io.dingodb.meta.InfoSchemaService;
137136
import io.dingodb.meta.MetaService;
@@ -195,6 +194,7 @@
195194
import java.util.Arrays;
196195
import java.util.Collections;
197196
import java.util.Comparator;
197+
import java.util.HashMap;
198198
import java.util.List;
199199
import java.util.Map;
200200
import java.util.Objects;
@@ -221,7 +221,6 @@
221221
import static io.dingodb.common.mysql.error.ErrorCode.ErrNotFoundDropTable;
222222
import static io.dingodb.common.mysql.error.ErrorCode.ErrNotSupportedYet;
223223
import static io.dingodb.common.mysql.error.ErrorCode.ErrPartitionMgmtOnNonpartitioned;
224-
import static io.dingodb.common.mysql.error.ErrorCode.ErrTruncatedWrongValue;
225224
import static io.dingodb.common.mysql.error.ErrorCode.ErrUnsupportedDDLOperation;
226225
import static io.dingodb.common.mysql.error.ErrorCode.ErrUnsupportedModifyVec;
227226
import static io.dingodb.common.util.NameCaseUtils.caseSensitive;
@@ -883,25 +882,7 @@ public void execute(@NonNull SqlAlterAddColumn sqlAlterAddColumn, CalcitePrepare
883882
throw new IllegalArgumentException("Add column, the engine must be transactional.");
884883
}
885884
}
886-
DingoSqlColumn dingoSqlColumn = (DingoSqlColumn) sqlAlterAddColumn.getColumnDeclaration();
887-
ColumnDefinition newColumn = fromSqlColumnDeclaration(
888-
dingoSqlColumn,
889-
new ContextSqlValidator(context, true),
890-
definition.keyColumns().stream().map(Column::getName).collect(Collectors.toList())
891-
);
892-
if (newColumn == null) {
893-
throw new RuntimeException("newColumn is null.");
894-
}
895-
896-
if (definition.getColumn(newColumn.getName()) != null) {
897-
throw DINGO_RESOURCE.duplicateColumn().ex();
898-
}
899-
if (dingoSqlColumn.isPrimaryKey()) {
900-
throw DINGO_RESOURCE.addColumnPrimaryError(newColumn.getName(), tableName).ex();
901-
}
902-
if (dingoSqlColumn.isAutoIncrement()) {
903-
throw DINGO_RESOURCE.addColumnAutoIncError(newColumn.getName(), tableName).ex();
904-
}
885+
ColumnDefinition newColumn = getAddColumn(context, sqlAlterAddColumn, definition, tableName);
905886
validateAddOrModifyColumn(newColumn, true);
906887
SqlIdentifier afterCol = sqlAlterAddColumn.getAfterCol();
907888
String afterColName;
@@ -2040,10 +2021,25 @@ public void execute(SqlBatchCreateTable sqlBatchCreateTable, CalcitePrepare.Cont
20402021
}
20412022
}
20422023

2024+
public void execute(SqlIdentifier table, List<AddingColInfo> addingColInfoList, CalcitePrepare.Context context) {
2025+
Pair<SubSnapshotSchema, String> schemaTableName
2026+
= getSchemaAndTableName(table, context);
2027+
final SubSnapshotSchema schema = Parameters.nonNull(schemaTableName.left, "table schema");
2028+
SchemaInfo schemaInfo = schema.getSchemaInfo(schema.getSchemaName());
2029+
String tableName = Parameters.nonNull(schemaTableName.right, "table name");
2030+
Table definition = schema.getTableInfo(tableName);
2031+
DdlService.root().addMultiColumn(schemaInfo, definition, addingColInfoList, "");
2032+
}
2033+
20432034
public void execute(SqlAlterTableOptions sqlAlterTableOptions, CalcitePrepare.Context context) {
20442035
LogUtils.info(log, "DDL execute:{}", sqlAlterTableOptions);
2045-
validateMultiSchemaChange(sqlAlterTableOptions, context);
20462036
List<SqlAlterTable> alterTableList = sqlAlterTableOptions.alterTableList;
2037+
List<AddingColInfo> addingColInfoList = mergeAddColumnDdl(alterTableList, context);
2038+
if (addingColInfoList != null) {
2039+
execute(sqlAlterTableOptions.table, addingColInfoList, context);
2040+
return;
2041+
}
2042+
validateMultiSchemaChange(sqlAlterTableOptions, context);
20472043
for (SqlAlterTable alterTable : alterTableList) {
20482044
int retry = 10;
20492045
while (retry-- > 0) {
@@ -3191,4 +3187,92 @@ public static void checkRenameTable(
31913187
}
31923188
}
31933189

3190+
public List<AddingColInfo> mergeAddColumnDdl(List<SqlAlterTable> alterTableList, CalcitePrepare.Context context) {
3191+
Map<String, String> tableMap = new HashMap<>();
3192+
List<AddingColInfo> addingColInfoList = new ArrayList<>();
3193+
for (int i = 0; i < alterTableList.size(); i ++) {
3194+
SqlAlterTable sqlAlterTable = alterTableList.get(i);
3195+
if (sqlAlterTable instanceof SqlAlterAddColumn) {
3196+
SqlAlterAddColumn sqlAlterAddColumn = (SqlAlterAddColumn) sqlAlterTable;
3197+
Pair<SubSnapshotSchema, String> schemaTableName
3198+
= getSchemaAndTableName(sqlAlterTable.table, context);
3199+
final SubSnapshotSchema schema = Parameters.nonNull(schemaTableName.left, "table schema");
3200+
SchemaInfo schemaInfo = schema.getSchemaInfo(schema.getSchemaName());
3201+
String tableName = Parameters.nonNull(schemaTableName.right, "table name");
3202+
tableName = convertName(tableName);
3203+
if (schemaInfo == null) {
3204+
if (context.getDefaultSchemaPath() != null && !context.getDefaultSchemaPath().isEmpty()) {
3205+
throw DINGO_RESOURCE.unknownSchema(context.getDefaultSchemaPath().get(0)).ex();
3206+
} else {
3207+
throw DINGO_RESOURCE.unknownSchema("DINGO").ex();
3208+
}
3209+
}
3210+
Table table = schema.getTableInfo(tableName);
3211+
if (table == null) {
3212+
throw DINGO_RESOURCE.unknownTable(schema.getSchemaName() + "." + tableName).ex();
3213+
}
3214+
ColumnDefinition newColumn = getAddColumn(context, sqlAlterAddColumn, table, tableName);
3215+
validateAddOrModifyColumn(newColumn, true);
3216+
SqlIdentifier afterCol = sqlAlterAddColumn.getAfterCol();
3217+
String afterColName;
3218+
if (afterCol != null) {
3219+
afterColName = afterCol.getSimple();
3220+
Column afterColumn = table.getColumns().stream()
3221+
.filter(col -> col.getSchemaState() == SchemaState.SCHEMA_PUBLIC
3222+
&& col.getName().equalsIgnoreCase(afterColName)).findFirst().orElse(null);
3223+
3224+
if (afterColumn == null && findAlterColumn(afterCol, i, alterTableList)) {
3225+
throw DINGO_RESOURCE.unknownColumn(afterColName, tableName).ex();
3226+
}
3227+
} else {
3228+
afterColName = null;
3229+
}
3230+
newColumn.setSchemaState(SchemaState.SCHEMA_PUBLIC);
3231+
3232+
AddingColInfo addingColInfo = AddingColInfo.builder()
3233+
.column(newColumn)
3234+
.afterColName(afterColName)
3235+
.firstCol(sqlAlterAddColumn.isFirstCol())
3236+
.build();
3237+
addingColInfoList.add(addingColInfo);
3238+
tableMap.put(schema.getSchemaName(), tableName);
3239+
} else {
3240+
return null;
3241+
}
3242+
}
3243+
return tableMap.size() == 1 ? addingColInfoList : null;
3244+
}
3245+
3246+
private static boolean findAlterColumn(SqlIdentifier afterCol, int index, List<SqlAlterTable> alterTableList) {
3247+
for (int i = 0; i < index; i++) {
3248+
SqlAlterAddColumn alterAddColumn = (SqlAlterAddColumn) alterTableList.get(i);
3249+
if (afterCol.equals(alterAddColumn.getAfterCol())) {
3250+
return true;
3251+
}
3252+
}
3253+
return false;
3254+
}
3255+
3256+
private static @NonNull ColumnDefinition getAddColumn(CalcitePrepare.Context context, SqlAlterAddColumn sqlAlterAddColumn, Table table, String tableName) {
3257+
DingoSqlColumn dingoSqlColumn = (DingoSqlColumn) sqlAlterAddColumn.getColumnDeclaration();
3258+
ColumnDefinition newColumn = fromSqlColumnDeclaration(
3259+
dingoSqlColumn,
3260+
new ContextSqlValidator(context, true),
3261+
table.keyColumns().stream().map(Column::getName).collect(Collectors.toList())
3262+
);
3263+
if (newColumn == null) {
3264+
throw new RuntimeException("newColumn is null.");
3265+
}
3266+
if (table.getColumn(newColumn.getName()) != null) {
3267+
throw DINGO_RESOURCE.duplicateColumn().ex();
3268+
}
3269+
if (dingoSqlColumn.isPrimaryKey()) {
3270+
throw DINGO_RESOURCE.addColumnPrimaryError(newColumn.getName(), tableName).ex();
3271+
}
3272+
if (dingoSqlColumn.isAutoIncrement()) {
3273+
throw DINGO_RESOURCE.addColumnAutoIncError(newColumn.getName(), tableName).ex();
3274+
}
3275+
return newColumn;
3276+
}
3277+
31943278
}

‎dingo-common/src/main/java/io/dingodb/common/ddl/ActionType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ public enum ActionType {
5252
ActionAlterIndex(64),
5353
ActionCreateTableAsQuery(65),
5454
ActionRefreshMeta(66),
55+
ActionAddMultiColumn(67),
5556
;
5657

5758
private final int code;

‎dingo-common/src/main/java/io/dingodb/common/ddl/DdlJob.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ public boolean mayNeedReorg() {
145145
|| actionType == ActionType.ActionDropTablePartition
146146
|| actionType == ActionType.ActionCreateTableAsQuery
147147
|| actionType == ActionType.ActionAddPrimaryKey
148+
|| actionType == ActionType.ActionAddMultiColumn
148149
) {
149150
return true;
150151
}
@@ -262,7 +263,7 @@ public String decodeArgs() {
262263
t = new TypeReference<List<String>>() {};
263264
} else if (actionType == ActionType.ActionDropColumn) {
264265
t = new TypeReference<List<String>>() {};
265-
} else if (actionType == ActionType.ActionAddColumn) {
266+
} else if (actionType == ActionType.ActionAddColumn || actionType == ActionType.ActionAddMultiColumn) {
266267
t = new TypeReference<List<AddingColInfo>>() {};
267268
} else if (actionType == ActionType.ActionRecoverTable || actionType == ActionType.ActionRecoverSchema) {
268269
t = new TypeReference<List<RecoverInfo>>() {};

‎dingo-executor/src/main/java/io/dingodb/server/executor/ddl/AbstractFiller.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ public boolean preWritePrimary(ReorgBackFillTask task) {
122122
indexTable = InfoSchemaService.root().getIndexDef(task.getTableId().domain, task.getTableId().seq,
123123
task.getIndexId().seq);
124124

125-
LogUtils.info(log, "index add filler tableId:{}, indexId:{}", task.getTableId(), task.getIndexId());
125+
LogUtils.info(log, "filler tableId:{}, indexId:{}", task.getTableId(), task.getIndexId());
126126
initFiller();
127127
columnIndices = table.getColumnIndices(indexTable.columns.stream()
128128
.map(Column::getName)

‎dingo-executor/src/main/java/io/dingodb/server/executor/ddl/AddColumnParam.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Copyright 2021 DataCanvas
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.dingodb.server.executor.ddl;
18+
19+
import io.dingodb.common.type.DingoType;
20+
import lombok.AllArgsConstructor;
21+
import lombok.Data;
22+
23+
@Data
24+
@AllArgsConstructor
25+
public class AddColumnParam {
26+
private Object defaultVal = null;
27+
private boolean nullable;
28+
private DingoType dingoType;
29+
private int addPos;
30+
}

0 commit comments

Comments
 (0)