From 2da088bf5e10b4ba1961298eef29f03abde3e5fd Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Thu, 19 Sep 2024 19:22:41 +0800 Subject: [PATCH 001/118] feat(external table): adapt external tables (#3195) * modify according to comments * finish syncing column for external table * remove redundant code * Modify external table judgment logic * external table does not support indexes and constraints * add sort test for repository * finish tests of TableService and DBIdentitiesService * modify error of test * modify format * add test for mulitple database sources * revert list-basic-schema-table-columns * modify according to comments * modify according to comments --- .../tools/dbbrowser/model/DBObjectType.java | 1 + .../tools/dbbrowser/model/DBTable.java | 2 + .../dbbrowser/schema/DBSchemaAccessor.java | 20 ++ .../schema/DBSchemaAccessorFactory.java | 16 +- .../schema/doris/DorisSchemaAccessor.java | 15 + .../MySQLNoLessThan5700SchemaAccessor.java | 21 ++ ...OBMySQLBetween400And432SchemaAccessor.java | 59 ++++ .../schema/mysql/OBMySQLSchemaAccessor.java | 65 ++++- .../mysql/ODPOBMySQLSchemaAccessor.java | 15 + ...racleBetween4000And4100SchemaAccessor.java | 2 +- ...BOracleBetween410And432SchemaAccessor.java | 95 +++++++ .../schema/oracle/OBOracleSchemaAccessor.java | 89 +++++- .../schema/oracle/OracleSchemaAccessor.java | 16 +- .../postgre/PostgresSchemaAccessor.java | 15 + .../ConnectionConfigRepositoryTest.java | 38 +++ .../dbobject/DBObjectRepositoryTest.java | 45 +++ .../service/db/DBIdentitiesServiceTest.java | 126 +++++++++ .../odc/service/db/DBTableServiceTest.java | 5 +- .../odc/service/db/TableServiceTest.java | 262 ++++++++++++++++++ ..._2_0_0__initialize_version_diff_config.sql | 4 + .../controller/v1/DBTableControllerV1.java | 3 +- .../web/controller/v2/DBSchemaController.java | 11 +- .../web/controller/v2/DBTableController.java | 6 +- .../logicaldatabase/LogicalTableService.java | 4 +- .../connection/table/TableService.java | 93 +++++-- .../table/model/QueryTableParams.java | 10 + .../service/connection/table/model/Table.java | 8 + .../datatransfer/DBObjectNameAccessor.java | 3 +- .../odc/service/db/DBIdentitiesService.java | 8 + .../odc/service/db/DBTableService.java | 18 +- ...eAndViewAndExternalTableColumnSyncer.java} | 4 +- .../syncer/object/DBExternalTableSyncer.java | 77 +++++ .../schema/syncer/object/DBTableSyncer.java | 3 +- .../partitionplan/PartitionPlanService.java | 10 +- .../processor/AbstractDlmPreprocessor.java | 3 +- .../session/ConnectConsoleService.java | 2 +- .../session/model/SqlExecuteResult.java | 41 ++- .../schema/api/TableExtensionPoint.java | 3 +- .../schema/mysql/MySQLTableExtension.java | 10 +- .../schema/obmysql/OBMySQLTableExtension.java | 31 ++- .../oboracle/OBOracleTableExtension.java | 24 +- .../schema/oracle/OracleTableExtension.java | 10 +- .../job/DorisScriptImportJob.java | 3 +- .../job/DorisTransferJobFactory.java | 3 +- .../job/MySQLSqlScriptImportJob.java | 2 +- .../job/factory/MySQLTransferJobFactory.java | 3 +- .../job/OracleSqlScriptImportJob.java | 2 +- .../job/factory/OracleTransferJobFactory.java | 3 +- 48 files changed, 1231 insertions(+), 78 deletions(-) create mode 100644 libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween400And432SchemaAccessor.java create mode 100644 libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java create mode 100644 server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBIdentitiesServiceTest.java create mode 100644 server/integration-test/src/test/java/com/oceanbase/odc/service/db/TableServiceTest.java rename server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/{DBTableAndViewColumnSyncer.java => DBTableAndViewAndExternalTableColumnSyncer.java} (93%) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalTableSyncer.java diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java index 7b2de36182..caeaa8f612 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java @@ -24,6 +24,7 @@ public enum DBObjectType { */ SCHEMA("SCHEMA"), TABLE("TABLE"), + EXTERNAL_TABLE("EXTERNAL TABLE"), LOGICAL_TABLE("LOGICAL TABLE"), COLUMN("COLUMN"), INDEX("INDEX"), diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTable.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTable.java index 605363d4b0..debb2f6ddb 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTable.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTable.java @@ -58,6 +58,8 @@ public class DBTable implements DBObject, DBObjectWarningDescriptor { private String warning; + private DBObjectType type; + @Data public static class DBTableOptions { private String charsetName; diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java index 21d68f020f..64439fb2f6 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java @@ -81,6 +81,26 @@ default List showTables(String schemaName) { */ List listTables(String schemaName, String tableNameLike); + /** + * Show all external table names list in the specified schema + */ + default List showExternalTables(String schemaName) { + return showExternalTablesLike(schemaName, null); + } + + List showExternalTablesLike(String schemaName, String tableNameLike); + + /** + * List all external table as BObjectIdentity in the specified schema + */ + List listExternalTables(String schemaName, String tableNameLike); + + /** + * Judge whether the table is an external table. If the current data source does not support + * external table feature,return false + */ + boolean isExternalTable(String schemaName, String tableName); + /** * Show all view names list in the specified schema */ diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorFactory.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorFactory.java index fd0bbbd6a0..616bf55b01 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorFactory.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorFactory.java @@ -30,10 +30,12 @@ import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLBetween220And225XSchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLBetween2260And2276SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLBetween2277And3XSchemaAccessor; +import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLBetween400And432SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLNoGreaterThan1479SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLSchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.mysql.ODPOBMySQLSchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.oracle.OBOracleBetween4000And4100SchemaAccessor; +import com.oceanbase.tools.dbbrowser.schema.oracle.OBOracleBetween410And432SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.oracle.OBOracleLessThan2270SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.oracle.OBOracleLessThan400SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.oracle.OBOracleSchemaAccessor; @@ -72,9 +74,12 @@ public DBSchemaAccessor buildForMySQL() { @Override public DBSchemaAccessor buildForOBMySQL() { Validate.notNull(this.dbVersion, "DBVersion can not be null"); - if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.0.0")) { - // OB version >= 4.0.0 + if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.3.2")) { + // OB version >= 4.3.2 return new OBMySQLSchemaAccessor(getJdbcOperations()); + } else if (VersionUtils.isGreaterThan(this.dbVersion, "4.0.0")) { + // OB version between [4.0.0, 4.3.2) + return new OBMySQLBetween400And432SchemaAccessor(getJdbcOperations()); } else if (VersionUtils.isGreaterThan(this.dbVersion, "2.2.76")) { // OB version between [2.2.77, 4.0.0) return new OBMySQLBetween2277And3XSchemaAccessor(getJdbcOperations()); @@ -105,9 +110,12 @@ public DBSchemaAccessor buildForOBMySQL() { @Override public DBSchemaAccessor buildForOBOracle() { Validate.notNull(this.dbVersion, "DBVersion can not be null"); - if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.1.0")) { - // OB version >= 4.1.0 + if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.3.2")) { + // OB version >= 4.3.2 return new OBOracleSchemaAccessor(getJdbcOperations(), new ALLDataDictTableNames()); + } else if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.1.0")) { + // OB version between [4.1.0, 4.3.2) + return new OBOracleBetween410And432SchemaAccessor(getJdbcOperations(), new ALLDataDictTableNames()); } else if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.0.0")) { // OB version between [4.0.0, 4.1.0) return new OBOracleBetween4000And4100SchemaAccessor(getJdbcOperations(), new ALLDataDictTableNames()); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java index 90ce3b070e..0842716d8e 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java @@ -225,6 +225,21 @@ public List listTables(String schemaName, String tableNameLike return results; } + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + return false; + } + protected List listBaseTables(String schemaName, String tableNameLike) throws DataAccessException { MySQLSqlBuilder sb = new MySQLSqlBuilder(); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java index 7301ab76e9..b0f1d84c9a 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java @@ -233,6 +233,27 @@ public List listTables(String schemaName, String tableNameLike return listBaseTables(schemaName, tableNameLike); } + @Override + public List showExternalTables(String schemaName) { + throw new UnsupportedOperationException("Not supported yet"); + } + + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + return false; + } + protected List listBaseTables(String schemaName, String tableNameLike) throws DataAccessException { MySQLSqlBuilder sb = new MySQLSqlBuilder(); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween400And432SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween400And432SchemaAccessor.java new file mode 100644 index 0000000000..477c273e58 --- /dev/null +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween400And432SchemaAccessor.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.tools.dbbrowser.schema.mysql; + +import java.util.List; + +import org.springframework.jdbc.core.JdbcOperations; + +import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; + +/** + * @description: applicable to OB [4.0.0,4.3.2) + * @author: zijia.cj + * @date: 2024/8/27 14:55 + * @since: 4.3.3 + */ +public class OBMySQLBetween400And432SchemaAccessor extends OBMySQLSchemaAccessor { + + + public OBMySQLBetween400And432SchemaAccessor(JdbcOperations jdbcOperations) { + super(jdbcOperations); + } + + @Override + public List showExternalTables(String schemaName) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + return false; + } +} diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java index 658d8f2dca..1ef531c7e7 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java @@ -25,6 +25,7 @@ import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; +import org.springframework.jdbc.core.BeanPropertyRowMapper; import org.springframework.jdbc.core.JdbcOperations; import com.oceanbase.tools.dbbrowser.model.DBColumnGroupElement; @@ -52,7 +53,7 @@ import lombok.extern.slf4j.Slf4j; /** - * 适用 OB 版本：[4.0.0, ~) + * 适用 OB 版本：[4.3.2, ~) * * @author jingtian */ @@ -361,6 +362,68 @@ public Map getTables(@NonNull String schemaName, List t return returnVal; } + @Override + public List showExternalTables(String schemaName) { + return showExternalTablesLike(schemaName, null); + } + + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + MySQLSqlBuilder sb = new MySQLSqlBuilder(); + sb.append("SELECT table_name FROM information_schema.tables WHERE TABLE_TYPE = 'EXTERNAL TABLE'"); + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND table_schema="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND table_name LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY table_name"); + return jdbcOperations.queryForList(sb.toString(), String.class); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + MySQLSqlBuilder sb = new MySQLSqlBuilder(); + sb.append("select table_schema as schema_name, 'EXTERNAL_TABLE' as type, table_name as name "); + sb.append("from information_schema.tables where table_type = 'EXTERNAL TABLE'"); + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND table_schema="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND table_name LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY schema_name, table_name"); + return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + MySQLSqlBuilder sb = new MySQLSqlBuilder(); + sb.append("SELECT table_type FROM information_schema.tables"); + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" Where table_schema="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableName)) { + sb.append(" AND table_name = "); + sb.value(tableName); + } + String tableType = jdbcOperations.queryForObject(sb.toString(), String.class); + if (tableType == null) { + throw new IllegalArgumentException("table name: " + tableName + " is not exist"); + } + if (StringUtils.equalsIgnoreCase(tableType, "EXTERNAL TABLE")) { + return true; + } else { + return false; + } + } + @Override protected void correctColumnPrecisionIfNeed(List tableColumns) {} } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/ODPOBMySQLSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/ODPOBMySQLSchemaAccessor.java index 5b341587c0..8ad6f47f37 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/ODPOBMySQLSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/ODPOBMySQLSchemaAccessor.java @@ -162,4 +162,19 @@ public DBProcedure getProcedure(String schemaName, String procedureName) { throw new UnsupportedOperationException("Not supported yet"); } + @Override + public List showExternalTables(String schemaName) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween4000And4100SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween4000And4100SchemaAccessor.java index f88aec2d4c..0a9e5a4894 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween4000And4100SchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween4000And4100SchemaAccessor.java @@ -27,7 +27,7 @@ * @author jingtian * @date 2024/4/15 */ -public class OBOracleBetween4000And4100SchemaAccessor extends OBOracleSchemaAccessor { +public class OBOracleBetween4000And4100SchemaAccessor extends OBOracleBetween410And432SchemaAccessor { public OBOracleBetween4000And4100SchemaAccessor(JdbcOperations jdbcOperations, OracleDataDictTableNames dataDictTableNames) { super(jdbcOperations, dataDictTableNames); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java new file mode 100644 index 0000000000..2996ba695b --- /dev/null +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.tools.dbbrowser.schema.oracle; + +import java.util.List; + +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.jdbc.core.JdbcOperations; + +import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.util.OracleDataDictTableNames; +import com.oceanbase.tools.dbbrowser.util.OracleSqlBuilder; +import com.oceanbase.tools.dbbrowser.util.StringUtils; + +/** + * @description: applicable to OB [4.1.0,4.3.2) + * @author: zijia.cj + * @date: 2024/8/27 15:13 + * @since: 4.3.3 + */ +public class OBOracleBetween410And432SchemaAccessor extends OBOracleSchemaAccessor { + + public OBOracleBetween410And432SchemaAccessor(JdbcOperations jdbcOperations, + OracleDataDictTableNames dataDictTableNames) { + super(jdbcOperations, dataDictTableNames); + } + + @Override + public List showExternalTables(String schemaName) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + @Override + public List showTablesLike(String schemaName, String tableNameLike) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("SELECT TABLE_NAME FROM "); + sb.append(dataDictTableNames.TABLES()); + sb.append(" WHERE OWNER="); + sb.value(schemaName); + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND TABLE_NAME LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY TABLE_NAME ASC"); + return jdbcOperations.queryForList(sb.toString(), String.class); + } + + @Override + public List listTables(String schemaName, String tableNameLike) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("select OWNER as schema_name, 'TABLE' as type,TABLE_NAME as name"); + sb.append(" from "); + sb.append(dataDictTableNames.TABLES()); + sb.append(" where 1=1 "); + + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND OWNER="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND TABLE_NAME LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY schema_name, type, name"); + return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); + } + +} diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java index 8d57ebd5f0..b42a09796b 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java @@ -85,7 +85,7 @@ import lombok.extern.slf4j.Slf4j; /** - * 适用于的 DB 版本：[4.1.0, ~) + * 适用于的 DB 版本：[4.3.2, ~) * * @author jingtian */ @@ -1035,4 +1035,91 @@ public Map getTables(@NonNull String schemaName, List t } return returnVal; } + + @Override + public List showExternalTables(String schemaName) { + return showExternalTablesLike(schemaName, null); + } + + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + return commonShowTablesLike(schemaName, tableNameLike, DBObjectType.EXTERNAL_TABLE); + } + + + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("select OWNER as schema_name, 'EXTERNAL_TABLE' as type,TABLE_NAME as name"); + sb.append(" from "); + sb.append(dataDictTableNames.TABLES()); + sb.append(" where EXTERNAL = 'YES'"); + + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND OWNER="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND TABLE_NAME LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY schema_name, type, name"); + return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); + } + + // After ob version 4.3.2, oracle model displaying table list needs to exclude external tables + @Override + public List showTablesLike(String schemaName, String tableNameLike) { + return commonShowTablesLike(schemaName, tableNameLike, DBObjectType.TABLE); + } + + // After ob version 4.3.2, oracle model displaying table list needs to exclude external tables + @Override + public List listTables(String schemaName, String tableNameLike) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("select OWNER as schema_name, 'TABLE' as type,TABLE_NAME as name"); + sb.append(" from "); + sb.append(dataDictTableNames.TABLES()); + sb.append(" where EXTERNAL = 'NO'"); + + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND OWNER="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND TABLE_NAME LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY schema_name, type, name"); + return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); + } + + private List commonShowTablesLike(String schemaName, String tableNameLike, + @NonNull DBObjectType tableType) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("SELECT TABLE_NAME FROM "); + sb.append(dataDictTableNames.TABLES()); + switch (tableType) { + case TABLE: + sb.append(" WHERE EXTERNAL = 'NO'"); + break; + case EXTERNAL_TABLE: + sb.append(" WHERE EXTERNAL = 'YES'"); + break; + default: + throw new UnsupportedOperationException("Not supported table type"); + } + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND OWNER="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND TABLE_NAME LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY TABLE_NAME ASC"); + return jdbcOperations.queryForList(sb.toString(), String.class); + } } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java index 35bba2e82d..4e787b63cc 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java @@ -247,6 +247,21 @@ public List listTables(String schemaName, String tableNameLike return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); } + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not support yet"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not support yet"); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + return false; + } + @Override public List listViews(String schemaName) { OracleSqlBuilder sb = new OracleSqlBuilder(); @@ -1835,5 +1850,4 @@ protected String getSynonymOwnerSymbol(DBSynonymType synonymType, String schemaN throw new UnsupportedOperationException("Not supported Synonym type"); } } - } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java index a4aa3be576..e75a823769 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java @@ -137,6 +137,21 @@ public List listTables(String schemaName, String tableNameLike throw new UnsupportedOperationException("Not supported yet"); } + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + return false; + } + @Override public List listViews(String schemaName) { throw new UnsupportedOperationException("Not supported yet"); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/ConnectionConfigRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/ConnectionConfigRepositoryTest.java index 228460d9e3..c5a124eb47 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/ConnectionConfigRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/ConnectionConfigRepositoryTest.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.metadb.connection; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -45,6 +46,8 @@ public class ConnectionConfigRepositoryTest extends ServiceTestEnv { private static final String CLUSTER_NAME = "C1"; private static final String TENANT_NAME = "T1"; private static final String USERNAME = "odcTest"; + private static final List INSERT_ORDER_NAME_LIST = Arrays.asList("C", "B", "A"); + private static final List LEXICAL_ORDER_NAME_LIST = Arrays.asList("A", "B", "C"); @Autowired private ConnectionConfigRepository repository; @@ -279,6 +282,41 @@ public void deleteByIds_servalConnsExist_deleteSucceed() { Assert.assertEquals(entities.size(), affectRows); } + @Test + public void findByOrganizationId_getListBySpecificOrganization_success() { + List entities = new ArrayList<>(); + for (String name : INSERT_ORDER_NAME_LIST) { + ConnectionEntity entity = createEntity(ConnectionVisibleScope.ORGANIZATION); + entity.setName(name); + entities.add(entity); + } + List savedEntities = repository.saveAll(entities); + List list = repository.findByOrganizationId( + ORGANIZATION_ID); + Assert.assertEquals(savedEntities.size(), list.size()); + for (int i = 0; i < list.size(); i++) { + Assert.assertEquals(list.get(i).getOrganizationId(), ORGANIZATION_ID); + } + } + + @Test + public void findByOrganizationIdOrderByNameAsc_getSortedListByNameAsc_success() { + List entities = new ArrayList<>(); + for (String name : INSERT_ORDER_NAME_LIST) { + ConnectionEntity entity = createEntity(ConnectionVisibleScope.ORGANIZATION); + entity.setName(name); + entities.add(entity); + } + List savedEntities = repository.saveAll(entities); + List list = repository.findByOrganizationIdOrderByNameAsc( + ORGANIZATION_ID); + Assert.assertEquals(savedEntities.size(), list.size()); + for (int i = 0; i < list.size(); i++) { + Assert.assertEquals(list.get(i).getOrganizationId(), ORGANIZATION_ID); + Assert.assertEquals(list.get(i).getName(), LEXICAL_ORDER_NAME_LIST.get(i)); + } + } + private ConnectionEntity createEntity(ConnectionVisibleScope visibleScope) { ConnectionEntity entity = TestRandom.nextObject(ConnectionEntity.class); entity.setId(null); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepositoryTest.java index cad40c1b9b..7c9c6b4c3c 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepositoryTest.java @@ -35,6 +35,11 @@ */ public class DBObjectRepositoryTest extends ServiceTestEnv { + private static final Long DATABASE_ID = 1L; + private static final DBObjectType DATASOURCE_TYPE = DBObjectType.SCHEMA; + private static final List INSERT_ORDER_NAME_LIST = Arrays.asList("C", "B", "A"); + private static final List LEXICAL_ORDER_NAME_LIST = Arrays.asList("A", "B", "C"); + @Autowired private DBObjectRepository dbObjectRepository; @@ -73,4 +78,44 @@ public void test_batchCreate() { Assert.assertEquals(entities.size(), saved.size()); } + @Test + public void findByDatabaseIdAndType_getListByDatabaseIdAndType_success() { + List entities = new ArrayList<>(); + for (String name : INSERT_ORDER_NAME_LIST) { + DBObjectEntity entity = TestRandom.nextObject(DBObjectEntity.class); + entity.setName(name); + entity.setDatabaseId(DATABASE_ID); + entity.setType(DATASOURCE_TYPE); + entities.add(entity); + } + List savedEntities = dbObjectRepository.saveAll(entities); + List list = dbObjectRepository.findByDatabaseIdAndType( + DATABASE_ID, DATASOURCE_TYPE); + Assert.assertEquals(savedEntities.size(), list.size()); + for (int i = 0; i < list.size(); i++) { + Assert.assertEquals(list.get(i).getDatabaseId(), DATABASE_ID); + Assert.assertEquals(list.get(i).getType(), DATASOURCE_TYPE); + } + } + + @Test + public void findByDatabaseIdAndTypeOrderByNameAsc_getListSortedByName_success() { + List entities = new ArrayList<>(); + for (String name : INSERT_ORDER_NAME_LIST) { + DBObjectEntity entity = TestRandom.nextObject(DBObjectEntity.class); + entity.setName(name); + entity.setDatabaseId(DATABASE_ID); + entity.setType(DATASOURCE_TYPE); + entities.add(entity); + } + List savedEntities = dbObjectRepository.saveAll(entities); + List list = dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc( + DATABASE_ID, DATASOURCE_TYPE); + Assert.assertEquals(savedEntities.size(), list.size()); + for (int i = 0; i < list.size(); i++) { + Assert.assertEquals(list.get(i).getDatabaseId(), DATABASE_ID); + Assert.assertEquals(list.get(i).getType(), DATASOURCE_TYPE); + Assert.assertEquals(list.get(i).getName(), LEXICAL_ORDER_NAME_LIST.get(i)); + } + } } diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBIdentitiesServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBIdentitiesServiceTest.java new file mode 100644 index 0000000000..84842489d0 --- /dev/null +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBIdentitiesServiceTest.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcOperations; + +import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.TestConnectionUtil; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionConstants; +import com.oceanbase.odc.core.session.ConnectionSessionUtil; +import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.service.db.model.ObjectIdentity; +import com.oceanbase.odc.service.db.model.SchemaIdentities; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/9/10 09:20 + * @since: 4.3.3 + */ +public class DBIdentitiesServiceTest extends ServiceTestEnv { + @Autowired + private DBIdentitiesService dbIdentitiesService; + + private final static List TABLE_NAME_LIST = Arrays.asList("test_table_1", "test_table_2", "test_table_3"); + private final static List VIEW_NAME_LIST = Arrays.asList("test_view_1", "test_view_2", "test_view_3"); + + @Before + public void setUp() { + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + StringBuilder tableBuilder = new StringBuilder(); + for (String name : TABLE_NAME_LIST) { + tableBuilder.append(String.format("CREATE TABLE IF NOT EXISTS %s (\n" + + " id BIGINT NOT NULL AUTO_INCREMENT,\n" + + " PRIMARY key (`id`)\n" + + ");", name)); + } + session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY).execute(tableBuilder.toString()); + StringBuilder viewBuilder = new StringBuilder(); + for (int i = 0; i < VIEW_NAME_LIST.size(); i++) { + viewBuilder.append(String.format("CREATE VIEW %s AS SELECT * FROM %s;", VIEW_NAME_LIST.get(i), + TABLE_NAME_LIST.get(i))); + } + session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY).execute(viewBuilder.toString()); + } + + @After + public void clear() { + ConnectionSession connectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + JdbcOperations jdbcOperations = + connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); + StringBuilder viewBuilder = new StringBuilder(); + for (String name : VIEW_NAME_LIST) { + viewBuilder.append(String.format("drop view %s;", name)); + } + jdbcOperations.execute(viewBuilder.toString()); + StringBuilder tableBuilder = new StringBuilder(); + for (String name : TABLE_NAME_LIST) { + tableBuilder.append(String.format("drop table %s;", name)); + } + jdbcOperations.execute(tableBuilder.toString()); + } + + @Test + public void list_whenTypesIsEmpty_getEmptyList() { + ConnectionSession connectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + List result = dbIdentitiesService.list(connectionSession, Collections.emptyList()); + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void list_whenTypesOnlyContainsTable_success() { + ConnectionSession connectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + List types = Collections.singletonList(DBObjectType.TABLE); + List result = dbIdentitiesService.list(connectionSession, types); + Assert.assertNotNull(result); + Map> schema2Identities = result.stream().collect( + Collectors.toMap(SchemaIdentities::getSchemaName, SchemaIdentities::getIdentities)); + List nameList = schema2Identities.get( + ConnectionSessionUtil.getCurrentSchema(connectionSession)).stream() + .filter(x -> DBObjectType.TABLE.equals(x.getType())).map(ObjectIdentity::getName).collect( + Collectors.toList()); + Assert.assertTrue(nameList.containsAll(TABLE_NAME_LIST)); + } + + @Test + public void list_whenTypesOnlyContainsView_success() { + ConnectionSession connectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + List types = Collections.singletonList(DBObjectType.VIEW); + List result = dbIdentitiesService.list(connectionSession, types); + Assert.assertNotNull(result); + Map> schema2Identities = result.stream().collect( + Collectors.toMap(SchemaIdentities::getSchemaName, SchemaIdentities::getIdentities)); + List nameList = schema2Identities.get( + ConnectionSessionUtil.getCurrentSchema(connectionSession)).stream() + .filter(x -> DBObjectType.VIEW.equals(x.getType())).map(ObjectIdentity::getName).collect( + Collectors.toList()); + Assert.assertTrue(nameList.containsAll(VIEW_NAME_LIST)); + } +} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBTableServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBTableServiceTest.java index 83c9da0c63..f0df673b27 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBTableServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBTableServiceTest.java @@ -25,6 +25,7 @@ import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.exception.NotFoundException; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; /** @@ -47,7 +48,7 @@ public void getTable_not_exist_OB_MYSQL() { ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); - DBTable table = dbTableService.getTable(session, "abc", "abc"); + DBTable table = dbTableService.getTable(session, "abc", "abc", DBObjectType.TABLE); } @Test @@ -57,6 +58,6 @@ public void getTable_not_exist_OB_ORACLE() { ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_ORACLE); - DBTable table = dbTableService.getTable(session, "abc", "abc"); + DBTable table = dbTableService.getTable(session, "abc", "abc", DBObjectType.TABLE); } } diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/TableServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/TableServiceTest.java new file mode 100644 index 0000000000..639cb63721 --- /dev/null +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/TableServiceTest.java @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.jdbc.core.JdbcOperations; + +import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.TestConnectionUtil; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionConstants; +import com.oceanbase.odc.core.session.ConnectionSessionUtil; +import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.core.shared.constant.OrganizationType; +import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor; +import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; +import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.connection.table.TableService; +import com.oceanbase.odc.service.connection.table.model.QueryTableParams; +import com.oceanbase.odc.service.connection.table.model.Table; +import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.iam.model.User; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/9/9 14:50 + * @since: 4.3.3 + */ +public class TableServiceTest extends ServiceTestEnv { + @Autowired + private TableService tableService; + @MockBean + private DBObjectRepository dbObjectRepository; + @MockBean + private AuthenticationFacade authenticationFacade; + @MockBean + private DatabaseService databaseService; + @MockBean + private DBResourcePermissionHelper dbResourcePermissionHelper; + + private static final String OB_MYSQL_TABLE_CREATE_TEMPLATE = "CREATE TABLE IF NOT EXISTS %s (\n" + + " id BIGINT NOT NULL AUTO_INCREMENT,\n" + + " PRIMARY key (`id`)\n" + + ")"; + + private static final String OB_ORACLE_TABLE_CREATE_TEMPLATE = "CREATE TABLE %s (\n" + + " id NUMBER PRIMARY KEY\n" + + ")"; + + private final static List TABLE_NAME_LIST = Arrays.asList("test_table_1", "test_table_2", "test_table_3"); + + private final static Long DATABASE_ID = 1L; + + private static final Long USER_ID = 1L; + + @BeforeClass + public static void setUp() { + createTablesByConnectType(ConnectType.OB_MYSQL, TABLE_NAME_LIST, OB_MYSQL_TABLE_CREATE_TEMPLATE); + createTablesByConnectType(ConnectType.OB_ORACLE, TABLE_NAME_LIST, OB_ORACLE_TABLE_CREATE_TEMPLATE); + createTablesByConnectType(ConnectType.MYSQL, TABLE_NAME_LIST, OB_MYSQL_TABLE_CREATE_TEMPLATE); + createTablesByConnectType(ConnectType.ORACLE, TABLE_NAME_LIST, OB_ORACLE_TABLE_CREATE_TEMPLATE); + } + + @AfterClass + public static void clear() { + dropTablesByConnectTypes(ConnectType.OB_MYSQL, TABLE_NAME_LIST); + dropTablesByConnectTypes(ConnectType.OB_ORACLE, TABLE_NAME_LIST); + dropTablesByConnectTypes(ConnectType.MYSQL, TABLE_NAME_LIST); + dropTablesByConnectTypes(ConnectType.ORACLE, TABLE_NAME_LIST); + } + + @Test + public void list_whenConnectionTypeIsOBMysqlAndTypesIsEmpty_getEmptyList() + throws SQLException, InterruptedException { + QueryTableParams params = QueryTableParams.builder() + .databaseId(DATABASE_ID) + .types(Collections.emptyList()) + .includePermittedAction(false) + .build(); + List list = tableService.list(params); + Assert.assertTrue(list.isEmpty()); + } + + @Test + public void list_whenConnectionTypeIsOBMysqlAndOrganizationTypeIsIndividual_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInIndividualSpace(ConnectType.OB_MYSQL); + } + + @Test + public void list_whenConnectionTypeIsOBMysqlAndOrganizationTypeIsTeam_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInTeamSpace(ConnectType.OB_MYSQL); + } + + @Test + public void list_whenConnectionTypeIsMysqlAndOrganizationTypeIsIndividual_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInIndividualSpace(ConnectType.MYSQL); + } + + @Test + public void list_whenConnectionTypeIsMysqlAndOrganizationTypeIsTeam_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInTeamSpace(ConnectType.MYSQL); + } + + @Test + public void list_whenConnectionTypeIsOBOracleAndOrganizationTypeIsIndividual_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInIndividualSpace(ConnectType.OB_ORACLE); + } + + @Test + public void list_whenConnectionTypeIsOBOracleAndOrganizationTypeIsTeam_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInTeamSpace(ConnectType.OB_ORACLE); + } + + @Test + public void list_whenConnectionTypeIsOracleAndOrganizationTypeIsIndividual_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInIndividualSpace(ConnectType.ORACLE); + } + + @Test + public void list_whenConnectionTypeIsOracleAndOrganizationTypeIsTeam_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInTeamSpace(ConnectType.ORACLE); + } + + private List getTableEntities() { + return TABLE_NAME_LIST.stream().map(name -> { + DBObjectEntity entity = new DBObjectEntity(); + entity.setName(name); + entity.setType(DBObjectType.TABLE); + entity.setDatabaseId(DATABASE_ID); + return entity; + }).collect(java.util.stream.Collectors.toList()); + } + + private User getIndivisualUser() { + User user = User.of(USER_ID); + user.setOrganizationType(OrganizationType.INDIVIDUAL); + return user; + } + + private User getTeamUser() { + User user = User.of(USER_ID); + user.setOrganizationType(OrganizationType.TEAM); + return user; + } + + private Database getDatabaseByConnectType(ConnectType connectType) { + Database database = new Database(); + database.setId(DATABASE_ID); + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(connectType); + ConnectionConfig connectionConfig = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(session); + String currentSchema = ConnectionSessionUtil.getCurrentSchema(session); + database.setName(currentSchema); + database.setDataSource(connectionConfig); + return database; + } + + private boolean containsAllIgnoreCase(List list, List toCheck) { + for (String item : toCheck) { + boolean found = false; + for (String listItem : list) { + if (listItem.equalsIgnoreCase(item)) { + found = true; + break; + } + } + if (!found) { + return false; + } + } + return true; + } + + private static void createTablesByConnectType(ConnectType connectType, List tableNames, String format) { + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(connectType); + SyncJdbcExecutor syncJdbcExecutor = session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); + for (String name : tableNames) { + syncJdbcExecutor.execute(String.format(format, name)); + } + } + + private static void dropTablesByConnectTypes(ConnectType connectType, List tableNames) { + ConnectionSession connectionSession = TestConnectionUtil.getTestConnectionSession(connectType); + JdbcOperations jdbcOperations = + connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); + for (String name : tableNames) { + jdbcOperations.execute(String.format("drop table %s", name)); + } + } + + private void testByConnectionTypeInIndividualSpace(ConnectType connectType) + throws SQLException, InterruptedException { + Mockito.when(authenticationFacade.currentUser()).thenReturn(getIndivisualUser()); + Mockito.when(databaseService.detail(Mockito.any())).thenReturn(getDatabaseByConnectType(connectType)); + QueryTableParams params = QueryTableParams.builder() + .databaseId(DATABASE_ID) + .types(Collections.singletonList(DBObjectType.TABLE)) + .includePermittedAction(false) + .build(); + List
list = tableService.list(params); + Assert.assertFalse(list.isEmpty()); + List nameList = list.stream().map(Table::getName).collect(Collectors.toList()); + Assert.assertTrue(containsAllIgnoreCase(nameList, TABLE_NAME_LIST)); + } + + private void testByConnectionTypeInTeamSpace(ConnectType obMysql) throws SQLException, InterruptedException { + Mockito.when(authenticationFacade.currentUser()).thenReturn(getTeamUser()); + Mockito.when(databaseService.detail(Mockito.any())).thenReturn(getDatabaseByConnectType(obMysql)); + Mockito.when(dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(Mockito.anyLong(), Mockito.any())) + .thenReturn(getTableEntities()); + Mockito.when(dbResourcePermissionHelper.getTablePermissions(Mockito.any())).thenReturn(new HashMap<>()); + QueryTableParams params = QueryTableParams.builder() + .databaseId(DATABASE_ID) + .types(Collections.singletonList(DBObjectType.TABLE)) + .includePermittedAction(false) + .build(); + List
list = tableService.list(params); + Assert.assertFalse(list.isEmpty()); + List nameList = list.stream().map(Table::getName).collect(Collectors.toList()); + Assert.assertTrue(containsAllIgnoreCase(nameList, TABLE_NAME_LIST)); + } + +} diff --git a/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql b/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql index 0adc9490b3..80d5e661ce 100644 --- a/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql +++ b/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql @@ -259,3 +259,7 @@ insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_column_group', 'OB_MYSQL', 'true', '4.3.0', CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_key`=`config_key`; insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_column_group', 'OB_ORACLE', 'true', '4.3.0', CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_key`=`config_key`; + +-- supports ob external table +insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_external_table', 'OB_MYSQL', 'true', '4.3.2', CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_key`=`config_key`; +insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_external_table', 'OB_ORACLE', 'true', '4.3.2', CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_key`=`config_key`; \ No newline at end of file diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableControllerV1.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableControllerV1.java index f1eeec011a..7c769b78c2 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableControllerV1.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableControllerV1.java @@ -35,6 +35,7 @@ import com.oceanbase.odc.service.session.ConnectSessionService; import com.oceanbase.odc.service.state.model.StateName; import com.oceanbase.odc.service.state.model.StatefulRoute; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; import io.swagger.annotations.ApiOperation; @@ -70,7 +71,7 @@ public OdcResult detail(@PathVariable String sid) { // parse sid and database name, sid:1-1:d:database:t:tb1 ResourceIdentifier i = ResourceIDParser.parse(sid); return OdcResult.ok(new OdcDBTable(tableService.getTable( - sessionService.nullSafeGet(i.getSid(), true), i.getDatabase(), i.getTable()))); + sessionService.nullSafeGet(i.getSid(), true), i.getDatabase(), i.getTable(), DBObjectType.TABLE))); } @ApiOperation(value = "getUpdateSql", notes = "获取修改表名的sql") diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java index 24d908a351..9d9586880e 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java @@ -16,7 +16,10 @@ package com.oceanbase.odc.server.web.controller.v2; import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -28,6 +31,7 @@ import com.oceanbase.odc.service.connection.table.TableService; import com.oceanbase.odc.service.connection.table.model.QueryTableParams; import com.oceanbase.odc.service.connection.table.model.Table; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import io.swagger.annotations.ApiOperation; @@ -48,10 +52,15 @@ public class DBSchemaController { @RequestMapping(value = "/tables", method = RequestMethod.GET) public ListResponse
list(@RequestParam(name = "databaseId") Long databaseId, @RequestParam(name = "includePermittedAction", required = false, - defaultValue = "false") boolean includePermittedAction) + defaultValue = "false") boolean includePermittedAction, + @RequestParam(required = false, name = "type") List types) throws SQLException, InterruptedException { + if (CollectionUtils.isEmpty(types)) { + types = Collections.singletonList(DBObjectType.TABLE); + } QueryTableParams params = QueryTableParams.builder() .databaseId(databaseId) + .types(types) .includePermittedAction(includePermittedAction) .build(); return Responses.list(tableService.list(params)); diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java index ccd0b8089d..8bbfe38344 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java @@ -40,6 +40,7 @@ import com.oceanbase.odc.service.session.ConnectSessionService; import com.oceanbase.odc.service.state.model.StateName; import com.oceanbase.odc.service.state.model.StatefulRoute; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBSchema; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.model.datatype.DataType; @@ -74,11 +75,12 @@ public ListResponse listTables(@PathVariable String sessionId, @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sessionId") public SuccessResponse getTable(@PathVariable String sessionId, @PathVariable(required = false) String databaseName, - @PathVariable String tableName) { + @PathVariable String tableName, + @RequestParam(required = false, name = "type", defaultValue = "TABLE") DBObjectType type) { Base64.Decoder decoder = Base64.getDecoder(); tableName = new String(decoder.decode(tableName)); ConnectionSession session = sessionService.nullSafeGet(sessionId, true); - return Responses.success(tableService.getTable(session, databaseName, tableName)); + return Responses.success(tableService.getTable(session, databaseName, tableName, type)); } @PostMapping(value = {"/{sessionId}/databases/{databaseName}/tables/generateCreateTableDDL", diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java index 235307ab0b..29f2ee557a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java @@ -274,8 +274,8 @@ public DetailLogicalTableResp detail(@NotNull Long logicalDatabaseId, @NotNull L TableExtensionPoint tableExtensionPoint = SchemaPluginUtil.getTableExtension(logicalDatabase.getConnectType().getDialectType()); if (tableExtensionPoint == null) { - throw new UnsupportedOperationException( - "Unsupported dialect " + logicalDatabase.getConnectType().getDialectType()); + throw new UnsupportedOperationException("the dialect " + logicalDatabase.getConnectType().getDialectType() + + " doesn't support the database object type " + DBObjectType.TABLE); } try (Connection connection = new DruidDataSourceFactory( connectionService.getForConnectionSkipPermissionCheck(physicalDatabase.getConnectionId())) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java index 7923a5ec29..cd9867f8b5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java @@ -19,6 +19,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; @@ -29,6 +30,7 @@ import java.util.stream.Collectors; import javax.validation.Valid; +import javax.validation.constraints.NotNull; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -54,6 +56,8 @@ import com.oceanbase.odc.service.connection.table.model.QueryTableParams; import com.oceanbase.odc.service.connection.table.model.Table; import com.oceanbase.odc.service.db.schema.DBSchemaSyncService; +import com.oceanbase.odc.service.db.schema.syncer.DBSchemaSyncer; +import com.oceanbase.odc.service.db.schema.syncer.object.DBExternalTableSyncer; import com.oceanbase.odc.service.db.schema.syncer.object.DBTableSyncer; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; @@ -87,6 +91,9 @@ public class TableService { @Autowired private DBTableSyncer dbTableSyncer; + @Autowired + private DBExternalTableSyncer dbExternalTableSyncer; + @Autowired private JdbcLockRegistry lockRegistry; @@ -99,37 +106,78 @@ public class TableService { @Transactional(rollbackFor = Exception.class) @SkipAuthorize("permission check inside") public List
list(@NonNull @Valid QueryTableParams params) throws SQLException, InterruptedException { + List types = params.getTypes(); + if (CollectionUtils.isEmpty(types)) { + return Collections.emptyList(); + } Database database = databaseService.detail(params.getDatabaseId()); ConnectionConfig dataSource = database.getDataSource(); OBConsoleDataSourceFactory factory = new OBConsoleDataSourceFactory(dataSource, true); + List
tables = new ArrayList<>(); try (SingleConnectionDataSource ds = (SingleConnectionDataSource) factory.getDataSource(); Connection conn = ds.getConnection()) { - TableExtensionPoint point = SchemaPluginUtil.getTableExtension(dataSource.getDialectType()); - Set latestTableNames = point.list(conn, database.getName()) - .stream().map(DBObjectIdentity::getName).collect(Collectors.toCollection(LinkedHashSet::new)); - if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { - return latestTableNames.stream().map(tableName -> { - Table table = new Table(); - table.setName(tableName); - table.setAuthorizedPermissionTypes(new HashSet<>(DatabasePermissionType.all())); - return table; - }).collect(Collectors.toList()); + TableExtensionPoint tableExtension = SchemaPluginUtil.getTableExtension(dataSource.getDialectType()); + if (tableExtension == null) { + throw new UnsupportedOperationException("the dialect " + dataSource.getDialectType() + + " doesn't support the database object type " + DBObjectType.TABLE); } - List tables = + if (types.contains(DBObjectType.TABLE)) { + generateListAndSyncDBTablesByTableType(params, database, dataSource, tables, conn, DBObjectType.TABLE, + tableExtension); + } + if (types.contains(DBObjectType.EXTERNAL_TABLE)) { + generateListAndSyncDBTablesByTableType(params, database, dataSource, tables, conn, + DBObjectType.EXTERNAL_TABLE, tableExtension); + } + return tables; + } + } + + private void generateListAndSyncDBTablesByTableType(QueryTableParams params, Database database, + ConnectionConfig dataSource, + List
tables, + Connection conn, DBObjectType tableType, TableExtensionPoint tableExtension) throws InterruptedException { + Set latestTableNames = tableExtension.list(conn, database.getName(), tableType) + .stream().map(DBObjectIdentity::getName).collect(Collectors.toCollection(LinkedHashSet::new)); + if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { + tables.addAll(latestTableNames.stream().map(tableName -> { + Table table = new Table(); + table.setName(tableName); + table.setAuthorizedPermissionTypes(new HashSet<>(DatabasePermissionType.all())); + table.setType(tableType); + return table; + }).collect(Collectors.toList())); + } else { + List existTables = dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(params.getDatabaseId(), - DBObjectType.TABLE); - Set existTableNames = tables.stream().map(DBObjectEntity::getName).collect(Collectors.toSet()); - if (latestTableNames.size() != existTableNames.size() || !existTableNames.containsAll(latestTableNames)) { - syncDBTables(conn, database, dataSource.getDialectType()); - tables = dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(params.getDatabaseId(), - DBObjectType.TABLE); + tableType); + Set existTableNames = + existTables.stream().map(DBObjectEntity::getName).collect(Collectors.toSet()); + if (latestTableNames.size() != existTableNames.size() + || !existTableNames.containsAll(latestTableNames)) { + syncDBTables(conn, database, dataSource.getDialectType(), getSyncerByTableType(tableType)); + existTables = + dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(params.getDatabaseId(), + tableType); } - return entitiesToModels(tables, database, params.getIncludePermittedAction()); + tables.addAll(entitiesToModels(existTables, database, params.getIncludePermittedAction())); + } + } + + private DBSchemaSyncer getSyncerByTableType(@NotNull DBObjectType tableType) { + switch (tableType) { + case TABLE: + return dbTableSyncer; + case EXTERNAL_TABLE: + return dbExternalTableSyncer; + default: + throw new IllegalArgumentException("Unsupported table type: " + tableType); } } - private void syncDBTables(Connection connection, Database database, DialectType dialectType) - throws InterruptedException { + private void syncDBTables(@NotNull Connection connection, @NotNull Database database, + @NotNull DialectType dialectType, + @NotNull DBSchemaSyncer syncer) throws InterruptedException { Lock lock = lockRegistry .obtain(dbSchemaSyncService.getSyncDBObjectLockKey(database.getDataSource().getId(), database.getId())); if (!lock.tryLock(3, TimeUnit.SECONDS)) { @@ -137,8 +185,8 @@ private void syncDBTables(Connection connection, Database database, DialectType new Object[] {ResourceType.ODC_TABLE.getLocalizedMessage()}, "Can not acquire jdbc lock"); } try { - if (dbTableSyncer.supports(dialectType)) { - dbTableSyncer.sync(connection, database, dialectType); + if (syncer.supports(dialectType)) { + syncer.sync(connection, database, dialectType); } else { throw new UnsupportedException("Unsupported dialect type: " + dialectType); } @@ -163,6 +211,7 @@ private List
entitiesToModels(Collection entities, Databa table.setCreateTime(entity.getCreateTime()); table.setUpdateTime(entity.getUpdateTime()); table.setOrganizationId(entity.getOrganizationId()); + table.setType(entity.getType()); if (includePermittedAction) { table.setAuthorizedPermissionTypes(id2Types.get(entity.getId())); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java index 78fe0e5eb7..441136438a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java @@ -15,8 +15,12 @@ */ package com.oceanbase.odc.service.connection.table.model; +import java.util.List; + import javax.validation.constraints.NotNull; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + import lombok.Builder; import lombok.Data; @@ -32,5 +36,11 @@ public class QueryTableParams { private Long databaseId; @NotNull private Boolean includePermittedAction; + /** + * table belonging to type in this collection needs to be fetched. if types is empty, return an + * empty collection. + */ + @NotNull + private List types; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java index 449946b7a6..fa5fd1d774 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java @@ -26,6 +26,7 @@ import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import lombok.Data; @@ -61,6 +62,13 @@ public class Table implements SecurityResource, OrganizationIsolated, Serializab @JsonProperty(access = Access.READ_ONLY) private Set authorizedPermissionTypes; + /** + * Used to distinguish between external and basic tables, and additional table types may be added + * later. + */ + @JsonProperty(access = Access.READ_ONLY) + private DBObjectType type; + @Override public String resourceId() { return this.id == null ? null : this.id.toString(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/DBObjectNameAccessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/DBObjectNameAccessor.java index 733858cd77..b3038a86c8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/DBObjectNameAccessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/DBObjectNameAccessor.java @@ -30,6 +30,7 @@ import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBSynonymType; import com.oceanbase.tools.loaddump.common.enums.ObjectType; @@ -93,7 +94,7 @@ public Set getObjectNames(@NonNull ObjectType objectType) { public Set getTableNames() { return queryNames(conn -> SchemaPluginUtil.getTableExtension(dialectType) - .list(conn, schema)).stream() + .list(conn, schema, DBObjectType.TABLE)).stream() .map(DBObjectIdentity::getName) .filter(name -> !StringUtils.endsWithIgnoreCase(name, OdcConstants.VALIDATE_DDL_TABLE_POSTFIX)) .collect(Collectors.toSet()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBIdentitiesService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBIdentitiesService.java index cc91e234df..895c3fc320 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBIdentitiesService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBIdentitiesService.java @@ -47,6 +47,9 @@ public List list(ConnectionSession session, List if (types.contains(DBObjectType.TABLE)) { listTables(schemaAccessor, all); } + if (types.contains(DBObjectType.EXTERNAL_TABLE)) { + listExternalTables(schemaAccessor, all); + } schemaAccessor.showDatabases().forEach(db -> all.computeIfAbsent(db, SchemaIdentities::of)); return new ArrayList<>(all.values()); } @@ -63,4 +66,9 @@ void listViews(DBSchemaAccessor schemaAccessor, Map al .forEach(i -> all.computeIfAbsent(i.getSchemaName(), SchemaIdentities::of).add(i)); } + void listExternalTables(DBSchemaAccessor schemaAccessor, Map all) { + schemaAccessor.listExternalTables(null, null) + .forEach(i -> all.computeIfAbsent(i.getSchemaName(), SchemaIdentities::of).add(i)); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java index a27c96c566..b1e21c20cf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java @@ -49,6 +49,7 @@ import com.oceanbase.odc.service.sqlcheck.SqlCheckUtil; import com.oceanbase.tools.dbbrowser.DBBrowser; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBSchema; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor; @@ -89,11 +90,18 @@ public List showTablesLike(@NotNull ConnectionSession session, String sc } public DBTable getTable(@NotNull ConnectionSession connectionSession, String schemaName, - @NotBlank String tableName) { + @NotBlank String tableName, @NotNull DBObjectType type) { DBSchemaAccessor schemaAccessor = DBSchemaAccessors.create(connectionSession); - PreConditions.validExists(ResourceType.OB_TABLE, "tableName", tableName, - () -> schemaAccessor.showTables(schemaName).stream().filter(name -> name.equals(tableName)) - .collect(Collectors.toList()).size() > 0); + if (type == DBObjectType.TABLE) { + PreConditions.validExists(ResourceType.OB_TABLE, "tableName", tableName, + () -> schemaAccessor.showTables(schemaName).stream().filter(name -> name.equals(tableName)) + .collect(Collectors.toList()).size() > 0); + } + if (type == DBObjectType.EXTERNAL_TABLE) { + PreConditions.validExists(ResourceType.OB_TABLE, "tableName", tableName, + () -> schemaAccessor.showExternalTables(schemaName).stream().filter(name -> name.equals(tableName)) + .collect(Collectors.toList()).size() > 0); + } try { return connectionSession.getSyncJdbcExecutor( ConnectionSessionConstants.BACKEND_DS_KEY) @@ -117,7 +125,7 @@ public Map getTables(@NotNull ConnectionSession connectionSessi public List listTables(@NotNull ConnectionSession connectionSession, String schemaName) { return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) .execute((ConnectionCallback>) con -> getTableExtensionPoint(connectionSession) - .list(con, schemaName)) + .list(con, schemaName, DBObjectType.TABLE)) .stream().map(item -> { DBTable table = new DBTable(); table.setName(item.getName()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewColumnSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewAndExternalTableColumnSyncer.java similarity index 93% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewColumnSyncer.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewAndExternalTableColumnSyncer.java index 7b63d5fc72..2ad6deb0e2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewColumnSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewAndExternalTableColumnSyncer.java @@ -36,7 +36,7 @@ * @date 2024/4/10 20:13 */ @Component -public class DBTableAndViewColumnSyncer extends AbstractDBColumnSyncer { +public class DBTableAndViewAndExternalTableColumnSyncer extends AbstractDBColumnSyncer { @Override Map> getLatestObjectToColumns(@NonNull ColumnExtensionPoint extensionPoint, @@ -48,7 +48,7 @@ Map> getLatestObjectToColumns(@NonNull ColumnExtensionPoint @Override public Collection getColumnRelatedObjectTypes() { - return Arrays.asList(DBObjectType.TABLE, DBObjectType.VIEW); + return Arrays.asList(DBObjectType.TABLE, DBObjectType.VIEW, DBObjectType.EXTERNAL_TABLE); } @Override diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalTableSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalTableSyncer.java new file mode 100644 index 0000000000..b687505d52 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalTableSyncer.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db.schema.syncer.object; + +import java.sql.Connection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.metadb.iam.PermissionEntity; +import com.oceanbase.odc.metadb.iam.PermissionRepository; +import com.oceanbase.odc.metadb.iam.UserPermissionRepository; +import com.oceanbase.odc.plugin.schema.api.TableExtensionPoint; +import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/8/23 14:33 + * @since: 4.3.3 + */ +@Component +public class DBExternalTableSyncer extends AbstractDBObjectSyncer { + + @Autowired + private PermissionRepository permissionRepository; + + @Autowired + private UserPermissionRepository userPermissionRepository; + + @Override + protected void preDelete(@NonNull Set toBeDeletedIds) { + List permissions = + permissionRepository.findByResourceTypeAndResourceIdIn(ResourceType.ODC_TABLE, toBeDeletedIds); + Set permissionIds = permissions.stream().map(PermissionEntity::getId).collect(Collectors.toSet()); + permissionRepository.deleteByIds(permissionIds); + userPermissionRepository.deleteByPermissionIds(permissionIds); + } + + @Override + public DBObjectType getObjectType() { + return DBObjectType.EXTERNAL_TABLE; + } + + @Override + Set getLatestObjectNames(@NonNull TableExtensionPoint extensionPoint, + @NonNull Connection connection, @NonNull Database database) { + return extensionPoint.list(connection, database.getName(), DBObjectType.EXTERNAL_TABLE).stream() + .map(DBObjectIdentity::getName).collect(Collectors.toSet()); + } + + @Override + Class getExtensionPointClass() { + return TableExtensionPoint.class; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java index 5e43b29870..bf2980f5e8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java @@ -59,7 +59,8 @@ protected void preDelete(@NonNull Set toBeDeletedIds) { @Override protected Set getLatestObjectNames(@NonNull TableExtensionPoint extensionPoint, @NonNull Connection connection, @NonNull Database database) { - return extensionPoint.list(connection, database.getName()).stream().map(DBObjectIdentity::getName) + return extensionPoint.list(connection, database.getName(), DBObjectType.TABLE).stream() + .map(DBObjectIdentity::getName) .collect(Collectors.toSet()); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/partitionplan/PartitionPlanService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/partitionplan/PartitionPlanService.java index ea6bc18c16..13952b9a5b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/partitionplan/PartitionPlanService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/partitionplan/PartitionPlanService.java @@ -69,6 +69,7 @@ import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.odc.service.plugin.TaskPluginUtil; import com.oceanbase.odc.service.session.ConnectSessionService; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.dbbrowser.model.DBTablePartition; @@ -118,7 +119,8 @@ public List getPartitionKeyDataTypes(@NonNull String sessionId, DialectType dialectType = connectionSession.getDialectType(); TableExtensionPoint tableExtensionPoint = SchemaPluginUtil.getTableExtension(dialectType); if (tableExtensionPoint == null) { - throw new UnsupportedOperationException("Unsupported dialect " + dialectType); + throw new UnsupportedOperationException("the dialect " + connectionSession.getConnectType().getDialectType() + + " doesn't support the database object type " + DBObjectType.TABLE); } DBTable dbTable = getJdbcOpt(connectionSession).execute((ConnectionCallback) con -> { try { @@ -274,7 +276,8 @@ public Map> generatePartitionDdl(@NonNull Co @NonNull PartitionPlanTableConfig tableConfig) throws Exception { TableExtensionPoint tableExtensionPoint = SchemaPluginUtil.getTableExtension(dialectType); if (tableExtensionPoint == null) { - throw new UnsupportedOperationException("Unsupported dialect " + dialectType); + throw new UnsupportedOperationException("the dialect " + dialectType + + " doesn't support the database object type " + DBObjectType.TABLE); } DBTable dbTable = tableExtensionPoint.getDetail(connection, schema, tableConfig.getTableName()); return generatePartitionDdl(connection, dialectType, dbTable, tableConfig); @@ -363,7 +366,8 @@ public String generatePartitionName(@NonNull Connection connection, @NonNull Dia @NonNull String schema, @NonNull PartitionPlanTableConfig tableConfig) throws Exception { TableExtensionPoint tableExtensionPoint = SchemaPluginUtil.getTableExtension(dialectType); if (tableExtensionPoint == null) { - throw new UnsupportedOperationException("Unsupported dialect " + dialectType); + throw new UnsupportedOperationException("the dialect " + dialectType + + " doesn't support the database object type " + DBObjectType.TABLE); } DBTable dbTable = tableExtensionPoint.getDetail(connection, schema, tableConfig.getTableName()); return generatePartitionName(connection, dialectType, dbTable, tableConfig); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java index ac97ab604c..4165d7c136 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java @@ -38,6 +38,7 @@ import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.odc.service.schedule.model.ScheduleChangeParams; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.util.MySQLSqlBuilder; import com.oceanbase.tools.dbbrowser.util.OracleSqlBuilder; import com.oceanbase.tools.dbbrowser.util.SqlBuilder; @@ -60,7 +61,7 @@ public List getAllTables(ConnectionSession sourceSession return Objects.requireNonNull(sourceSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) .execute((ConnectionCallback>) con -> SchemaPluginUtil.getTableExtension( sourceSession.getDialectType()) - .list(con, schemaName))) + .list(con, schemaName, DBObjectType.TABLE))) .stream().map(o -> { DataArchiveTableConfig config = new DataArchiveTableConfig(); config.setTableName(o.getName()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java index 8ad8e50ff3..6ea4a55956 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java @@ -583,7 +583,7 @@ private SqlExecuteResult generateResult(@NonNull ConnectionSession connectionSes log.warn("Failed to init sql type", e); } try (TraceStage s = watch.start(SqlExecuteStages.INIT_EDITABLE_INFO)) { - resultTable = result.initEditableInfo(); + resultTable = result.initEditableInfo(connectionSession, cxt); } catch (Exception e) { log.warn("Failed to init editable info", e); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java index f201357557..ad34e2758b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java @@ -30,6 +30,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang.StringUtils; @@ -38,6 +39,7 @@ import com.oceanbase.odc.common.util.ExceptionUtils; import com.oceanbase.odc.common.util.TraceStage; import com.oceanbase.odc.common.util.TraceWatch; +import com.oceanbase.odc.common.util.VersionUtils; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.core.shared.constant.DialectType; @@ -52,6 +54,7 @@ import com.oceanbase.odc.core.sql.parser.AbstractSyntaxTree; import com.oceanbase.odc.core.sql.parser.AbstractSyntaxTreeFactories; import com.oceanbase.odc.service.common.util.PLObjectErrMsgUtils; +import com.oceanbase.odc.service.db.browser.DBSchemaAccessors; import com.oceanbase.odc.service.feature.AllFeatures; import com.oceanbase.odc.service.session.model.OdcResultSetMetaData.OdcTable; import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; @@ -63,6 +66,7 @@ import com.oceanbase.tools.dbbrowser.parser.result.BasicResult; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor; +import cn.hutool.core.collection.CollectionUtil; import lombok.Data; import lombok.Getter; import lombok.NoArgsConstructor; @@ -82,6 +86,8 @@ @NoArgsConstructor @Slf4j public class SqlExecuteResult { + public static final String SHOW_EXTERNAL_TABLES_IN_SCHEMA = "SHOW_EXTERNAL_TABLES_IN_SCHEMA"; + public static final String MIN_OB_VERSION_FOR_EXTERNAL_TABLE = "4.3.2"; private List columnLabels; private List columns; private SqlExecuteStatus status = SqlExecuteStatus.CREATED; @@ -135,8 +141,9 @@ public void initWarningMessage(ConnectionSession connectionSession) { } } - public OdcTable initEditableInfo() { + public OdcTable initEditableInfo(@NonNull ConnectionSession connectionSession, @NonNull Map cxt) { boolean editable = true; + editable = !checkContainsExternalTable(connectionSession, cxt); OdcTable resultTable = null; Set relatedTablesOrViews = new HashSet<>(); if (Objects.isNull(this.resultSetMetaData)) { @@ -204,6 +211,38 @@ public OdcTable initEditableInfo() { return resultTable; } + private boolean checkContainsExternalTable(@NonNull ConnectionSession connectionSession, + @NonNull Map cxt) { + DialectType dialectType = connectionSession.getDialectType(); + if (dialectType == DialectType.OB_MYSQL || dialectType == DialectType.OB_ORACLE) { + String obVersion = ConnectionSessionUtil.getVersion(connectionSession); + if (VersionUtils.isGreaterThanOrEqualsTo(obVersion, MIN_OB_VERSION_FOR_EXTERNAL_TABLE)) { + List columnList = resultSetMetaData.getFieldMetaDataList(); + Map schemaAndTable2Column = columnList.stream() + .collect(Collectors.groupingBy(jcmd -> jcmd.getCatalogName() + "." + jcmd.getTableName(), + Collectors.collectingAndThen(Collectors.toList(), + lst -> lst.get(0)))); + Set columnSet = new HashSet<>(schemaAndTable2Column.values()); + for (JdbcColumnMetaData columnMetaData : columnSet) { + String catalogName = columnMetaData.getCatalogName(); + String tableName = columnMetaData.getTableName(); + Map> schema2ExternalTables = + (Map>) cxt.computeIfAbsent(SHOW_EXTERNAL_TABLES_IN_SCHEMA, + k -> new HashMap<>()); + List externalTables = schema2ExternalTables.computeIfAbsent(catalogName, k -> { + DBSchemaAccessor schemaAccessor = DBSchemaAccessors.create(connectionSession); + return schemaAccessor.showExternalTables(catalogName); + }); + if (CollectionUtil.contains(externalTables, tableName)) { + return true; + } + } + } + } + return false; + } + + public void initColumnInfo(@NonNull ConnectionSession connectionSession, OdcTable resultTable, @NonNull DBSchemaAccessor schemaAccessor) { if (this.resultSetMetaData == null) { diff --git a/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java b/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java index e0658823c1..25db79b650 100644 --- a/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java +++ b/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java @@ -21,6 +21,7 @@ import org.pf4j.ExtensionPoint; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; /** @@ -32,7 +33,7 @@ */ public interface TableExtensionPoint extends ExtensionPoint { - List list(Connection connection, String schemaName); + List list(Connection connection, String schemaName, DBObjectType tableType); List showNamesLike(Connection connection, String schemaName, String tableNameLike); diff --git a/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java b/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java index 06bbe4c002..f8a14973ce 100644 --- a/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java +++ b/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java @@ -22,6 +22,7 @@ import com.oceanbase.odc.plugin.schema.mysql.utils.DBAccessorUtil; import com.oceanbase.odc.plugin.schema.obmysql.OBMySQLTableExtension; import com.oceanbase.tools.dbbrowser.editor.DBTableEditor; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor; import com.oceanbase.tools.dbbrowser.stats.DBStatsAccessor; @@ -44,9 +45,14 @@ public DBTable getDetail(@NonNull Connection connection, @NonNull String schemaN table.setOwner(schemaName); table.setName(tableName); table.setColumns(schemaAccessor.listTableColumns(schemaName, tableName)); - table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); table.setPartition(schemaAccessor.getPartition(schemaName, tableName)); - table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); + if (!schemaAccessor.isExternalTable(schemaName, tableName)) { + table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); + table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); + table.setType(DBObjectType.TABLE); + } else { + table.setType(DBObjectType.EXTERNAL_TABLE); + } table.setDDL(schemaAccessor.getTableDDL(schemaName, tableName)); table.setTableOptions(schemaAccessor.getTableOptions(schemaName, tableName)); table.setStats(getTableStats(connection, schemaName, tableName)); diff --git a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java index fec24ae055..c3fd04f1a3 100644 --- a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java +++ b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java @@ -47,10 +47,26 @@ public class OBMySQLTableExtension implements TableExtensionPoint { @Override - public List list(@NonNull Connection connection, @NonNull String schemaName) { - return getSchemaAccessor(connection).showTables(schemaName).stream().map(item -> { + public List list(@NonNull Connection connection, @NonNull String schemaName, + @NonNull DBObjectType tableType) { + List nameList; + switch (tableType) { + case TABLE: + nameList = getSchemaAccessor(connection).showTables(schemaName); + return generateDBObjectIdentityByTableType(schemaName, nameList, DBObjectType.TABLE); + case EXTERNAL_TABLE: + nameList = getSchemaAccessor(connection).showExternalTables(schemaName); + return generateDBObjectIdentityByTableType(schemaName, nameList, DBObjectType.EXTERNAL_TABLE); + default: + throw new IllegalArgumentException("Unsupported table type: " + tableType); + } + } + + private List generateDBObjectIdentityByTableType(String schemaName, List nameList, + DBObjectType tableType) { + return nameList.stream().map(item -> { DBObjectIdentity identity = new DBObjectIdentity(); - identity.setType(DBObjectType.TABLE); + identity.setType(tableType); identity.setSchemaName(schemaName); identity.setName(item); return identity; @@ -74,9 +90,14 @@ public DBTable getDetail(@NonNull Connection connection, @NonNull String schemaN table.setOwner(schemaName); table.setName(tableName); table.setColumns(schemaAccessor.listTableColumns(schemaName, tableName)); - table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); + if (!schemaAccessor.isExternalTable(schemaName, tableName)) { + table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); + table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); + table.setType(DBObjectType.TABLE); + } else { + table.setType(DBObjectType.EXTERNAL_TABLE); + } table.setPartition(parser.getPartition()); - table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); table.setDDL(ddl); table.setTableOptions(schemaAccessor.getTableOptions(schemaName, tableName)); table.setStats(getTableStats(connection, schemaName, tableName)); diff --git a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java index bc9e78218a..193fc3c654 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java +++ b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java @@ -30,6 +30,7 @@ import com.oceanbase.odc.plugin.schema.oboracle.utils.DBAccessorUtil; import com.oceanbase.tools.dbbrowser.editor.DBTableEditor; import com.oceanbase.tools.dbbrowser.model.DBConstraintType; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.model.DBTable.DBTableOptions; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; @@ -66,16 +67,21 @@ public DBTable getDetail(@NonNull Connection connection, @NonNull String schemaN table.setOwner(schemaName); table.setName(tableName); table.setColumns(columns); - /** - * If the constraint name cannot be obtained through ddl of the table, then the constraint - * information will still be obtained through DBSchemaAccessor - */ - List constraints = parser.listConstraints(); - table.setConstraints(constraints.stream().anyMatch(c -> Objects.isNull(c.getName())) - ? accessor.listTableConstraints(schemaName, tableName) - : constraints); + if (!accessor.isExternalTable(schemaName, tableName)) { + /** + * If the constraint name cannot be obtained through ddl of the table, then the constraint + * information will still be obtained through DBSchemaAccessor + */ + List constraints = parser.listConstraints(); + table.setConstraints(constraints.stream().anyMatch(c -> Objects.isNull(c.getName())) + ? accessor.listTableConstraints(schemaName, tableName) + : constraints); + table.setIndexes(parser.listIndexes()); + table.setType(DBObjectType.TABLE); + } else { + table.setType(DBObjectType.EXTERNAL_TABLE); + } table.setPartition(parser.getPartition()); - table.setIndexes(parser.listIndexes()); DBTableOptions tableOptions = accessor.getTableOptions(schemaName, tableName); table.setTableOptions(tableOptions); table.setDDL(getTableDDL(connection, schemaName, tableName, parser, columns, tableOptions)); diff --git a/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java b/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java index d0ee22b295..1c05686373 100644 --- a/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java +++ b/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java @@ -24,6 +24,7 @@ import com.oceanbase.odc.plugin.schema.oboracle.OBOracleTableExtension; import com.oceanbase.odc.plugin.schema.oracle.utils.DBAccessorUtil; import com.oceanbase.tools.dbbrowser.editor.DBTableEditor; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.model.DBTableStats; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor; @@ -47,9 +48,14 @@ public DBTable getDetail(@NonNull Connection connection, @NonNull String schemaN table.setOwner(schemaName); table.setName(tableName); table.setColumns(schemaAccessor.listTableColumns(schemaName, tableName)); - table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); table.setPartition(schemaAccessor.getPartition(schemaName, tableName)); - table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); + if (!schemaAccessor.isExternalTable(schemaName, tableName)) { + table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); + table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); + table.setType(DBObjectType.TABLE); + } else { + table.setType(DBObjectType.EXTERNAL_TABLE); + } table.setDDL(schemaAccessor.getTableDDL(schemaName, tableName)); table.setTableOptions(schemaAccessor.getTableOptions(schemaName, tableName)); table.setStats(getTableStats(connection, schemaName, tableName)); diff --git a/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisScriptImportJob.java b/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisScriptImportJob.java index 95644e354d..2af1779ce0 100644 --- a/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisScriptImportJob.java +++ b/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisScriptImportJob.java @@ -63,7 +63,8 @@ protected boolean isObjectExists() throws SQLException { DBObjectIdentity target = DBObjectIdentity.of(object.getSchema(), DBObjectType.getEnumByName(object.getType()), object.getName()); try (Connection conn = dataSource.getConnection()) { - List tables = new DorisTableExtension().list(conn, object.getSchema()); + List tables = + new DorisTableExtension().list(conn, object.getSchema(), DBObjectType.TABLE); return CollectionUtils.containsAny(tables, target); } } diff --git a/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisTransferJobFactory.java b/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisTransferJobFactory.java index f1a447e10e..4c69f220f8 100644 --- a/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisTransferJobFactory.java +++ b/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisTransferJobFactory.java @@ -33,6 +33,7 @@ import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.datax.model.JobContent.Parameter; import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.datax.model.parameter.MySQLWriterPluginParameter; import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.factory.BaseTransferJobFactory; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.loaddump.common.enums.ObjectType; @@ -55,7 +56,7 @@ protected List queryTableColumns(Connection connection, ObjectRes @Override protected List queryTransferObjects(Connection connection, boolean transferDDL) { - return new DorisTableExtension().list(connection, transferConfig.getSchemaName()).stream() + return new DorisTableExtension().list(connection, transferConfig.getSchemaName(), DBObjectType.TABLE).stream() .map(table -> new DataTransferObject(ObjectType.TABLE, table.getName())) .collect(Collectors.toList()); } diff --git a/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/MySQLSqlScriptImportJob.java b/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/MySQLSqlScriptImportJob.java index d340ad3d88..0227e87138 100644 --- a/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/MySQLSqlScriptImportJob.java +++ b/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/MySQLSqlScriptImportJob.java @@ -63,7 +63,7 @@ protected boolean isObjectExists() throws SQLException { try (Connection conn = dataSource.getConnection()) { switch (object.getType()) { case "TABLE": - objects = new MySQLTableExtension().list(conn, object.getSchema()); + objects = new MySQLTableExtension().list(conn, object.getSchema(), DBObjectType.TABLE); break; case "VIEW": objects = new MySQLViewExtension().list(conn, object.getSchema()); diff --git a/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/factory/MySQLTransferJobFactory.java b/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/factory/MySQLTransferJobFactory.java index 4ba2e3c955..6625e34ed1 100644 --- a/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/factory/MySQLTransferJobFactory.java +++ b/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/factory/MySQLTransferJobFactory.java @@ -35,6 +35,7 @@ import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.MySQLSqlScriptImportJob; import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.datax.DataXTransferJob; import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.datax.model.JobConfiguration; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.loaddump.common.enums.ObjectType; @@ -58,7 +59,7 @@ protected List queryTableColumns(Connection connection, ObjectRes @Override protected List queryTransferObjects(Connection connection, boolean transferDDL) { List objects = new ArrayList<>(); - new MySQLTableExtension().list(connection, transferConfig.getSchemaName()) + new MySQLTableExtension().list(connection, transferConfig.getSchemaName(), DBObjectType.TABLE) .forEach(table -> objects.add(new DataTransferObject(ObjectType.TABLE, table.getName()))); if (transferDDL) { new MySQLViewExtension().list(connection, transferConfig.getSchemaName()) diff --git a/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/OracleSqlScriptImportJob.java b/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/OracleSqlScriptImportJob.java index 1fda4f8c18..43216ebea0 100644 --- a/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/OracleSqlScriptImportJob.java +++ b/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/OracleSqlScriptImportJob.java @@ -72,7 +72,7 @@ protected boolean isObjectExists() throws SQLException { ObjectType type = ObjectType.valueOfName(object.getType()); switch (type) { case TABLE: - objects = new OracleTableExtension().list(conn, object.getSchema()); + objects = new OracleTableExtension().list(conn, object.getSchema(), DBObjectType.TABLE); break; case VIEW: objects = new OracleViewExtension().list(conn, object.getSchema()); diff --git a/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/factory/OracleTransferJobFactory.java b/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/factory/OracleTransferJobFactory.java index 513586026f..2c77ef1a43 100644 --- a/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/factory/OracleTransferJobFactory.java +++ b/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/factory/OracleTransferJobFactory.java @@ -46,6 +46,7 @@ import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.factory.BaseTransferJobFactory; import com.oceanbase.odc.plugin.task.oracle.datatransfer.job.OracleSchemaExportJob; import com.oceanbase.odc.plugin.task.oracle.datatransfer.job.OracleSqlScriptImportJob; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBSynonymType; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.dbbrowser.util.OracleSqlBuilder; @@ -70,7 +71,7 @@ protected List queryTableColumns(Connection connection, ObjectRes @Override protected List queryTransferObjects(Connection connection, boolean transferDDL) { List objects = new ArrayList<>(); - new OracleTableExtension().list(connection, transferConfig.getSchemaName()) + new OracleTableExtension().list(connection, transferConfig.getSchemaName(), DBObjectType.TABLE) .forEach(table -> objects.add(new DataTransferObject(ObjectType.TABLE, table.getName()))); if (transferDDL) { new OracleViewExtension().list(connection, transferConfig.getSchemaName()) From acf70aaa14eb0ad1ef964fc521ddcda2d9635931 Mon Sep 17 00:00:00 2001 From: guowl3 Date: Mon, 23 Sep 2024 15:50:57 +0800 Subject: [PATCH 002/118] fix(dlm): task status is wrong (#3491) * fix the issue of wrong task status * format * fix the issue of throw not found exception when database is deleted. * simplify code --- .../oceanbase/odc/service/dlm/DLMService.java | 23 +++++++++++------ .../ScheduleResponseMapperFactory.java | 25 +++++++++++-------- .../listener/DefaultJobTerminateListener.java | 2 +- .../task/processor/DLMResultProcessor.java | 6 +++-- 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java index 5d3571eaf4..20da69850c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java @@ -116,21 +116,28 @@ public List findByScheduleTaskId(Long scheduleTaskId) { Collectors.toList()); } + /** + * generate final task status by scheduleTaskId when the task is finished + */ @SkipAuthorize("odc internal usage") - public TaskStatus getTaskStatus(Long scheduleTaskId) { - return getTaskStatus(findByScheduleTaskId(scheduleTaskId)); - } - - public TaskStatus getTaskStatus(List dlmTableUnits) { + public TaskStatus getFinalTaskStatus(Long scheduleTaskId) { + List dlmTableUnits = findByScheduleTaskId(scheduleTaskId); Set collect = dlmTableUnits.stream().map(DlmTableUnit::getStatus).collect( Collectors.toSet()); + // If any table fails, the task is considered a failure. if (collect.contains(TaskStatus.FAILED)) { return TaskStatus.FAILED; } - if (collect.contains(TaskStatus.DONE) && collect.size() == 1) { - return TaskStatus.DONE; + // If any table is canceled, the task is considered canceled. + if (collect.contains(TaskStatus.CANCELED)) { + return TaskStatus.CANCELED; + } + // The task is considered failed if any table is still preparing or running when the task is + // finished. + if (collect.contains(TaskStatus.PREPARING) || collect.contains(TaskStatus.RUNNING)) { + return TaskStatus.FAILED; } - return TaskStatus.CANCELED; + return TaskStatus.DONE; } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java index bec162afcd..dca87f0f89 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java @@ -34,6 +34,7 @@ import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.constant.FlowStatus; import com.oceanbase.odc.core.shared.constant.TaskType; +import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.metadb.flow.FlowInstanceEntity; import com.oceanbase.odc.metadb.flow.FlowInstanceRepository; import com.oceanbase.odc.metadb.flow.UserTaskInstanceEntity; @@ -195,11 +196,7 @@ public ScheduleDetailRespHist generateHistoryScheduleDetail(Schedule schedule) { resp.setUpdateTime(schedule.getUpdateTime()); resp.setDescription(schedule.getDescription()); resp.setJobParameters(detailParameters(schedule)); - - List databaseByIds = getDatabaseByIds(Collections.singleton(schedule.getDatabaseId())); - if (!databaseByIds.isEmpty()) { - resp.setDatabase(databaseByIds.get(0)); - } + resp.setDatabase(detailDatabaseOrNull(schedule.getDatabaseId())); Set approvableFlowInstanceIds = approvalPermissionService.getApprovableApprovalInstances() .stream() @@ -387,24 +384,24 @@ private ScheduleTaskParameters detailParameters(Schedule schedule) { switch (schedule.getType()) { case DATA_ARCHIVE: { DataArchiveParameters parameters = (DataArchiveParameters) schedule.getParameters(); - parameters.setSourceDatabase(databaseService.detail(parameters.getSourceDatabaseId())); - parameters.setTargetDatabase(databaseService.detail(parameters.getTargetDataBaseId())); + parameters.setSourceDatabase(detailDatabaseOrNull(parameters.getSourceDatabaseId())); + parameters.setTargetDatabase(detailDatabaseOrNull(parameters.getTargetDataBaseId())); limiterService.findByScheduleId(schedule.getId()).ifPresent(parameters::setRateLimit); return parameters; } case DATA_DELETE: { DataDeleteParameters parameters = (DataDeleteParameters) schedule.getParameters(); if (parameters.getTargetDatabaseId() != null) { - parameters.setTargetDatabase(databaseService.detail(parameters.getTargetDatabaseId())); + parameters.setTargetDatabase(detailDatabaseOrNull(parameters.getTargetDatabaseId())); } - parameters.setDatabase(databaseService.detail(parameters.getDatabaseId())); + parameters.setDatabase(detailDatabaseOrNull(parameters.getDatabaseId())); limiterService.findByScheduleId(schedule.getId()).ifPresent(parameters::setRateLimit); return parameters; } case SQL_PLAN: { SqlPlanParameters parameters = (SqlPlanParameters) schedule.getParameters(); if (parameters.getDatabaseId() != null) { - parameters.setDatabaseInfo(databaseService.detail(parameters.getDatabaseId())); + parameters.setDatabaseInfo(detailDatabaseOrNull(parameters.getDatabaseId())); } return parameters; } @@ -412,4 +409,12 @@ private ScheduleTaskParameters detailParameters(Schedule schedule) { return schedule.getParameters(); } } + + private Database detailDatabaseOrNull(Long databaseId) { + try { + return databaseService.detail(databaseId); + } catch (NotFoundException e) { + return null; + } + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobTerminateListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobTerminateListener.java index 7680674290..d00e1b6841 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobTerminateListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobTerminateListener.java @@ -63,7 +63,7 @@ public class DefaultJobTerminateListener extends AbstractEventListener { - TaskStatus taskStatus = "DLM".equals(jobEntity.getJobType()) ? dlmService.getTaskStatus(o.getId()) + TaskStatus taskStatus = "DLM".equals(jobEntity.getJobType()) ? dlmService.getFinalTaskStatus(o.getId()) : event.getStatus().convertTaskStatus(); scheduleTaskService.updateStatusById(o.getId(), taskStatus); log.info("Update schedule task status to {} succeed,scheduleTaskId={}", taskStatus, o.getId()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/DLMResultProcessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/DLMResultProcessor.java index ae44a56626..0bfb1cae6e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/DLMResultProcessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/DLMResultProcessor.java @@ -60,8 +60,10 @@ public void process(TaskResult result) { log.info("Create or update dlm tableUnits success,jobIdentity={},scheduleTaskId={}", result.getJobIdentity(), dlmTableUnits.get(0).getScheduleTaskId()); - TaskStatus taskStatus = dlmService.getTaskStatus(dlmTableUnits); - taskService.updateStatusById(dlmTableUnits.get(0).getScheduleTaskId(), taskStatus); + TaskStatus taskStatus = taskService.nullSafeGetById(dlmTableUnits.get(0).getScheduleTaskId()).getStatus(); + if (taskStatus != TaskStatus.RUNNING) { + taskService.updateStatusById(dlmTableUnits.get(0).getScheduleTaskId(), TaskStatus.RUNNING); + } log.info("Update schedule task status to {} success", taskStatus); } catch (Exception e) { log.warn("Refresh result failed.", e); From fa1993ac10278a77b6d098f3b8fd4528e8c6ce75 Mon Sep 17 00:00:00 2001 From: guowl3 Date: Tue, 24 Sep 2024 14:34:21 +0800 Subject: [PATCH 003/118] fix(schedule): risk level mismatch when operating a schedule (#3529) * fix select risklevel failed * fix description --- .../flow/util/DescriptionGenerator.java | 8 +++---- .../odc/service/schedule/ScheduleService.java | 14 +++++------ .../flowtask/DefaultApprovalFlowClient.java | 7 ++++++ .../schedule/model/CreateScheduleReq.java | 23 ------------------- .../schedule/model/ScheduleChangeParams.java | 20 ++++++++++++++++ .../processor/ScheduleChangePreprocessor.java | 10 +++++--- 6 files changed, 45 insertions(+), 37 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/DescriptionGenerator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/DescriptionGenerator.java index 7d0c93dfdc..422dcbd76d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/DescriptionGenerator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/DescriptionGenerator.java @@ -23,7 +23,7 @@ import com.oceanbase.odc.service.databasechange.model.DatabaseChangeDatabase; import com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq; import com.oceanbase.odc.service.flow.task.model.MultipleDatabaseChangeParameters; -import com.oceanbase.odc.service.schedule.model.CreateScheduleReq; +import com.oceanbase.odc.service.schedule.model.ScheduleChangeParams; /** * @Author：tinker @@ -51,10 +51,10 @@ public static void generateDescription(CreateFlowInstanceReq req) { } } - public static void generateScheduleDescription(CreateScheduleReq req) { - if (StringUtils.isEmpty(req.getDescription())) { + public static void generateScheduleDescription(ScheduleChangeParams req) { + if (StringUtils.isEmpty(req.getCreateScheduleReq().getDescription())) { String descFormat = "[%s]%s.%s"; - req.setDescription(String.format(descFormat, + req.getCreateScheduleReq().setDescription(String.format(descFormat, req.getEnvironmentName(), req.getConnectionName(), req.getDatabaseName())); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 1fac0aef9e..319e34c9d3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -221,7 +221,7 @@ public List dispatchCreateSchedule(CreateFlowInstanceReq createScheduleReq.setParameters(parameters.getScheduleTaskParameters()); createScheduleReq.setTriggerConfig(parameters.getTriggerConfig()); createScheduleReq.setType(parameters.getType()); - createScheduleReq.setDescription(parameters.getDescription()); + createScheduleReq.setDescription(createReq.getDescription()); scheduleChangeParams = ScheduleChangeParams.with(createScheduleReq); break; } @@ -230,7 +230,7 @@ public List dispatchCreateSchedule(CreateFlowInstanceReq updateScheduleReq.setParameters(parameters.getScheduleTaskParameters()); updateScheduleReq.setTriggerConfig(parameters.getTriggerConfig()); updateScheduleReq.setType(parameters.getType()); - updateScheduleReq.setDescription(parameters.getDescription()); + updateScheduleReq.setDescription(createReq.getDescription()); scheduleChangeParams = ScheduleChangeParams.with(parameters.getTaskId(), updateScheduleReq); break; } @@ -258,8 +258,8 @@ public ChangeScheduleResp changeSchedule(ScheduleChangeParams req) { ScheduleEntity entity = new ScheduleEntity(); entity.setName(req.getCreateScheduleReq().getName()); - entity.setProjectId(req.getCreateScheduleReq().getProjectId()); - DescriptionGenerator.generateScheduleDescription(req.getCreateScheduleReq()); + entity.setProjectId(req.getProjectId()); + DescriptionGenerator.generateScheduleDescription(req); entity.setDescription(req.getCreateScheduleReq().getDescription()); entity.setJobParametersJson(JsonUtils.toJson(req.getCreateScheduleReq().getParameters())); entity.setTriggerConfigJson(JsonUtils.toJson(req.getCreateScheduleReq().getTriggerConfig())); @@ -271,9 +271,9 @@ public ChangeScheduleResp changeSchedule(ScheduleChangeParams req) { entity.setOrganizationId(authenticationFacade.currentOrganizationId()); entity.setCreatorId(authenticationFacade.currentUserId()); entity.setModifierId(authenticationFacade.currentUserId()); - entity.setDatabaseId(req.getCreateScheduleReq().getDatabaseId()); - entity.setDatabaseName(req.getCreateScheduleReq().getDatabaseName()); - entity.setDataSourceId(req.getCreateScheduleReq().getConnectionId()); + entity.setDatabaseId(req.getDatabaseId()); + entity.setDatabaseName(req.getDatabaseName()); + entity.setDataSourceId(req.getConnectionId()); targetSchedule = scheduleMapper.entityToModel(scheduleRepository.save(entity)); req.setScheduleId(targetSchedule.getId()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DefaultApprovalFlowClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DefaultApprovalFlowClient.java index a6a8f7b870..7861dc8c3f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DefaultApprovalFlowClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/flowtask/DefaultApprovalFlowClient.java @@ -45,6 +45,13 @@ public Long create(ScheduleChangeParams params) { : params.getUpdateScheduleReq().getTriggerConfig()); } req.setParameters(alterScheduleParameters); + req.setProjectId(params.getProjectId()); + req.setProjectName(params.getProjectName()); + req.setDatabaseName(params.getDatabaseName()); + req.setConnectionId(params.getConnectionId()); + req.setConnectionName(params.getConnectionName()); + req.setEnvironmentId(params.getEnvironmentId()); + req.setEnvironmentName(params.getEnvironmentName()); return SpringContextUtil.getBean(FlowInstanceService.class).createAlterSchedule(req); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/CreateScheduleReq.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/CreateScheduleReq.java index 7dbe966971..754292698d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/CreateScheduleReq.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/CreateScheduleReq.java @@ -18,8 +18,6 @@ import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonProperty.Access; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonTypeInfo.As; @@ -27,7 +25,6 @@ import com.oceanbase.odc.service.dlm.model.DataArchiveParameters; import com.oceanbase.odc.service.dlm.model.DataDeleteParameters; import com.oceanbase.odc.service.loaddata.model.LoadDataParameters; -import com.oceanbase.odc.service.schedule.processor.ScheduleChangePreprocessor; import com.oceanbase.odc.service.sqlplan.model.SqlPlanParameters; import lombok.Data; @@ -66,25 +63,5 @@ public class CreateScheduleReq { private String description; - /** - * Followings are filled by aspect {@link ScheduleChangePreprocessor} - */ - @JsonProperty(access = Access.READ_ONLY) - private Long projectId; - @JsonProperty(access = Access.READ_ONLY) - private String projectName; - @JsonProperty(access = Access.READ_ONLY) - private Long databaseId; - @JsonProperty(access = Access.READ_ONLY) - private String databaseName; - @JsonProperty(access = Access.READ_ONLY) - private Long connectionId; - @JsonProperty(access = Access.READ_ONLY) - private String connectionName; - @JsonProperty(access = Access.READ_ONLY) - private Long environmentId; - @JsonProperty(access = Access.READ_ONLY) - private String environmentName; - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleChangeParams.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleChangeParams.java index 39d353664d..80d1b7040f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleChangeParams.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleChangeParams.java @@ -15,6 +15,9 @@ */ package com.oceanbase.odc.service.schedule.model; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty.Access; + import lombok.Data; /** @@ -36,6 +39,23 @@ public class ScheduleChangeParams { private UpdateScheduleReq updateScheduleReq; + @JsonProperty(access = Access.READ_ONLY) + private Long projectId; + @JsonProperty(access = Access.READ_ONLY) + private String projectName; + @JsonProperty(access = Access.READ_ONLY) + private Long databaseId; + @JsonProperty(access = Access.READ_ONLY) + private String databaseName; + @JsonProperty(access = Access.READ_ONLY) + private Long connectionId; + @JsonProperty(access = Access.READ_ONLY) + private String connectionName; + @JsonProperty(access = Access.READ_ONLY) + private Long environmentId; + @JsonProperty(access = Access.READ_ONLY) + private String environmentName; + public static ScheduleChangeParams with(Long id, OperationType type) { ScheduleChangeParams req = new ScheduleChangeParams(); req.setScheduleId(id); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java index 236dbc2a60..fad0339aa4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java @@ -65,7 +65,7 @@ public void process(ScheduleChangeParams params) { ScheduleType type; if (params.getOperationType() == OperationType.CREATE) { type = params.getCreateScheduleReq().getType(); - adaptCreateScheduleReq(params.getCreateScheduleReq()); + adaptScheduleChangeParams(params); } else { type = scheduleService.nullSafeGetModelById(params.getScheduleId()).getType(); } @@ -93,7 +93,7 @@ public void afterPropertiesSet() throws Exception { }); } - private void adaptCreateScheduleReq(CreateScheduleReq req) { + private void adaptScheduleChangeParams(ScheduleChangeParams req) { Database srcDb = databaseService.detail(getTargetDatabaseId(req)); req.setProjectId(srcDb.getProject().getId()); req.setProjectName(srcDb.getProject().getName()); @@ -105,7 +105,11 @@ private void adaptCreateScheduleReq(CreateScheduleReq req) { req.setDatabaseId(srcDb.getId()); } - private Long getTargetDatabaseId(CreateScheduleReq req) { + private Long getTargetDatabaseId(ScheduleChangeParams params) { + if (params.getOperationType() != OperationType.CREATE) { + return scheduleService.nullSafeGetById(params.getScheduleId()).getDatabaseId(); + } + CreateScheduleReq req = params.getCreateScheduleReq(); switch (req.getType()) { case DATA_ARCHIVE: { DataArchiveParameters parameters = (DataArchiveParameters) req.getParameters(); From e5db37f5842a4d2734a1c212048d7847c0c9b0f9 Mon Sep 17 00:00:00 2001 From: guowl3 Date: Tue, 24 Sep 2024 19:46:29 +0800 Subject: [PATCH 004/118] fix(sqlplan): generate task failed (#3535) * fix execute sqlplan * fix execute sqlplan --- .../odc/service/quartz/OdcJobListener.java | 10 ++++++- .../service/quartz/OdcTriggerListener.java | 1 - .../odc/service/schedule/ScheduleService.java | 30 +++++++++++++++++-- .../odc/service/schedule/job/SqlPlanJob.java | 9 ++---- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/OdcJobListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/OdcJobListener.java index 009e12e521..a86b671f32 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/OdcJobListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/OdcJobListener.java @@ -17,12 +17,14 @@ import java.util.Objects; import java.util.Optional; +import java.util.Set; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.quartz.JobKey; import org.quartz.JobListener; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import com.oceanbase.odc.common.json.JsonUtils; @@ -69,6 +71,8 @@ public class OdcJobListener implements JobListener { private HostProperties hostProperties; @Autowired private LatestTaskMappingRepository latestTaskMappingRepository; + @Value("${odc.iam.auth.type}") + protected Set authType; private static final String ODC_JOB_LISTENER = "ODC_JOB_LISTENER"; @@ -101,7 +105,11 @@ public void jobToBeExecuted(JobExecutionContext context) { userEntity.setOrganizationId(scheduleEntity.getOrganizationId()); User taskCreator = new User(userEntity); SecurityContextUtils.setCurrentUser(taskCreator); - + if (scheduleEntity.getType() == ScheduleType.PARTITION_PLAN + || (!authType.contains("obcloud") && scheduleEntity.getType() == ScheduleType.SQL_PLAN)) { + log.info("Skip preparing tasks for partition plan or sql plan,and create flow task later."); + return; + } // Create or load task. Long targetTaskId = ScheduleTaskUtils.getTargetTaskId(context); ScheduleTaskEntity entity; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/OdcTriggerListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/OdcTriggerListener.java index f1976bd497..8c8647396c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/OdcTriggerListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/OdcTriggerListener.java @@ -76,7 +76,6 @@ public boolean vetoJobExecution(Trigger trigger, JobExecutionContext context) { log.warn("The job will be skipped, job key:" + trigger.getJobKey()); ScheduleAlarmUtils.misfire(Long.parseLong(trigger.getJobKey().getName()), new Date()); } - log.info("The job will be execution,job key:" + trigger.getJobKey()); return skipExecution; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 319e34c9d3..1a6279fcd9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -564,7 +564,18 @@ private boolean isInvalidSchedule(ScheduleEntity schedule) { public void stopTask(Long scheduleId, Long scheduleTaskId) { nullSafeGetByIdWithCheckPermission(scheduleId, true); - scheduleTaskService.stop(scheduleTaskId); + Lock lock = jdbcLockRegistry.obtain(getScheduleTaskLockKey(scheduleTaskId)); + try { + if (!lock.tryLock(10, TimeUnit.SECONDS)) { + throw new ConflictException(ErrorCodes.ResourceModifying, "Can not acquire jdbc lock"); + } + scheduleTaskService.stop(scheduleTaskId); + } catch (InterruptedException e) { + log.error("Stop task failed", e); + throw new ConflictException(ErrorCodes.ResourceModifying, "Can not acquire jdbc lock"); + } finally { + lock.unlock(); + } } /** @@ -573,7 +584,18 @@ public void stopTask(Long scheduleId, Long scheduleTaskId) { */ public void startTask(Long scheduleId, Long scheduleTaskId) { nullSafeGetByIdWithCheckPermission(scheduleId, true); - scheduleTaskService.start(scheduleTaskId); + Lock lock = jdbcLockRegistry.obtain(getScheduleTaskLockKey(scheduleTaskId)); + try { + if (!lock.tryLock(10, TimeUnit.SECONDS)) { + throw new ConflictException(ErrorCodes.ResourceModifying, "Can not acquire jdbc lock"); + } + scheduleTaskService.start(scheduleTaskId); + } catch (InterruptedException e) { + log.error("Start task failed", e); + throw new ConflictException(ErrorCodes.ResourceModifying, "Can not acquire jdbc lock"); + } finally { + lock.unlock(); + } } public void rollbackTask(Long scheduleId, Long scheduleTaskId) { @@ -995,4 +1017,8 @@ private String getLogicalDatabaseChangeActionLockKey(@NonNull String executionId private String getScheduleChangeLockKey(@NonNull Long scheduleId) { return "schedule-change-" + scheduleId; } + + private String getScheduleTaskLockKey(@NonNull Long scheduleTaskId) { + return "schedule-task-" + scheduleTaskId; + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/SqlPlanJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/SqlPlanJob.java index 355f2fdb26..4fc48beba0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/SqlPlanJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/SqlPlanJob.java @@ -19,7 +19,6 @@ import java.util.List; import java.util.Map; -import org.quartz.JobDataMap; import org.quartz.JobExecutionContext; import com.alibaba.fastjson.JSON; @@ -90,13 +89,11 @@ public void execute(JobExecutionContext context) { executeInTaskFramework(context); return; } - - JobDataMap jobDataMap = context.getJobDetail().getJobDataMap(); - - ScheduleEntity scheduleEntity = JSON.parseObject(JSON.toJSONString(jobDataMap), ScheduleEntity.class); - + ScheduleEntity scheduleEntity = scheduleService.nullSafeGetById(ScheduleTaskUtils.getScheduleId(context)); DatabaseChangeParameters taskParameters = JsonUtils.fromJson(scheduleEntity.getJobParametersJson(), DatabaseChangeParameters.class); + log.info("Execute sql plan job, scheduleId={}, taskParameters={}", scheduleEntity.getId(), + JSON.toJSONString(taskParameters)); taskParameters.setParentScheduleType(ScheduleType.SQL_PLAN); CreateFlowInstanceReq flowInstanceReq = new CreateFlowInstanceReq(); flowInstanceReq.setParameters(taskParameters); From 0adf3b359dbc11ac7be90a563134a6293e009875 Mon Sep 17 00:00:00 2001 From: LioRoger Date: Thu, 26 Sep 2024 15:52:45 +0800 Subject: [PATCH 005/118] feat(osc): let osc task resumable (#3516) * dispatch osc away from flow task, make osc resumable * fix integrate test case * fix comment * fix comment * correct status and progress compute logic * correct status and progress compute logic * correct status and progress compute logic --- .../config/SystemConfigRepositoryTest.java | 43 ++- .../odc/metadb/task/JobRepositoryTest.java | 4 + .../flow/FlowResponseMapperFactoryTest.java | 6 + .../OnlineSchemaChangeExpiredTest.java | 2 +- .../DumperResultSetExportTaskManagerTest.java | 51 ++-- .../oceanbase/odc/common/json/JsonUtils.java | 3 + .../odc/core/shared/constant/FlowStatus.java | 4 + .../odc/core/shared/constant/TaskStatus.java | 3 + .../web/controller/v2/OscController.java | 7 + .../flow/BeanInjectedClassDelegate.java | 16 ++ .../odc/service/flow/FlowInstanceService.java | 19 +- .../flow/task/BaseODCFlowTaskDelegate.java | 13 +- .../task/BaseRuntimeFlowableDelegate.java | 2 +- .../OnlineSchemaChangeFlowableTask.java | 232 +++++++++++----- .../onlineschemachange/OscService.java | 80 +++++- .../model/OnlineSchemaChangeParameters.java | 5 + .../oms/OmsRequestUtil.java | 117 -------- .../oms/client/BaseOmsClient.java | 6 +- .../oscfms/ActionScheduler.java | 6 +- .../oscfms/OscActionFsmBase.java | 151 ++++++---- .../action/oms/OmsCleanResourcesAction.java | 14 +- .../action/oms/OmsModifyDataTaskAction.java | 2 +- .../action/oms/OmsMonitorDataTaskAction.java | 4 +- .../oscfms/action/oms/OmsSwapTableAction.java | 2 +- .../schedule/model/ScheduleMapper.java | 3 + .../OnlineSchemaChangeFlowableTaskTest.java | 123 +++++++++ .../oms/client/OmsClientTest.java | 258 ++++++++++++++++++ .../oscfms/OscActionFsmBaseTest.java | 155 +++++++++++ 28 files changed, 1030 insertions(+), 301 deletions(-) delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/OmsRequestUtil.java create mode 100644 server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java create mode 100644 server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oms/client/OmsClientTest.java create mode 100644 server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBaseTest.java diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/config/SystemConfigRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/config/SystemConfigRepositoryTest.java index 633f8f80f6..01ac99f153 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/config/SystemConfigRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/config/SystemConfigRepositoryTest.java @@ -18,33 +18,33 @@ import java.util.List; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.jdbc.JdbcTestUtils; import com.oceanbase.odc.ServiceTestEnv; +import lombok.extern.slf4j.Slf4j; + /** * @author liuyizhuo.lyz * @date 2024/3/21 */ +@Slf4j public class SystemConfigRepositoryTest extends ServiceTestEnv { @Autowired private SystemConfigRepository systemConfigRepository; - @Before - public void setUp() { - systemConfigRepository.getJdbcTemplate().batchUpdate("delete from config_system_configuration"); - } - @Test public void test_Insert_NotExists() { - systemConfigRepository.getJdbcTemplate().batchUpdate("delete from config_system_configuration"); - systemConfigRepository.insert(getConfigEntity()); - Assert.assertEquals(1, JdbcTestUtils.countRowsInTable(systemConfigRepository.getJdbcTemplate(), + int prevRows = JdbcTestUtils.countRowsInTable(systemConfigRepository.getJdbcTemplate(), + "config_system_configuration"); + int inserted = systemConfigRepository.insert(getConfigEntity()); + Assert.assertEquals(inserted, 1); + Assert.assertEquals(prevRows + 1, JdbcTestUtils.countRowsInTable(systemConfigRepository.getJdbcTemplate(), "config_system_configuration")); + delete("dummy.key"); } @Test @@ -57,14 +57,18 @@ public void test_Insert_Exists() { SystemConfigEntity entity = systemConfigRepository.queryByKey("dummy.key"); Assert.assertEquals("value", entity.getValue()); + delete("dummy.key"); } @Test public void test_Upsert_NotExists() { - systemConfigRepository.getJdbcTemplate().batchUpdate("delete from config_system_configuration"); - systemConfigRepository.upsert(getConfigEntity()); - Assert.assertEquals(1, JdbcTestUtils.countRowsInTable(systemConfigRepository.getJdbcTemplate(), + int prevRows = JdbcTestUtils.countRowsInTable(systemConfigRepository.getJdbcTemplate(), + "config_system_configuration"); + int inserted = systemConfigRepository.upsert(getConfigEntity()); + Assert.assertEquals(inserted, 1); + Assert.assertEquals(prevRows + 1, JdbcTestUtils.countRowsInTable(systemConfigRepository.getJdbcTemplate(), "config_system_configuration")); + delete("dummy.key"); } @Test @@ -77,6 +81,7 @@ public void test_Upsert_Exists() { SystemConfigEntity entity = systemConfigRepository.queryByKey("dummy.key"); Assert.assertEquals("value1", entity.getValue()); + delete("dummy.key"); } @Test @@ -88,6 +93,8 @@ public void test_QueryByKeyPrefix() { List entities = systemConfigRepository.queryByKeyPrefix("dummy"); Assert.assertEquals(2, entities.size()); + delete("dummy.key"); + delete("dummy.key1"); } @Test @@ -95,12 +102,24 @@ public void test_QueryByKey() { systemConfigRepository.upsert(getConfigEntity()); SystemConfigEntity entity = systemConfigRepository.queryByKey("dummy.key"); Assert.assertEquals("value", entity.getValue()); + delete("dummy.key"); + } + + private void delete(String... keys) { + for (String key : keys) { + int rows = systemConfigRepository.getJdbcTemplate() + .update("delete from `config_system_configuration` where `key` = '" + key + "'"); + log.info("delete {} rows with key = {}", rows, key); + } } private SystemConfigEntity getConfigEntity() { SystemConfigEntity entity = new SystemConfigEntity(); entity.setKey("dummy.key"); entity.setValue("value"); + entity.setLabel("master"); + entity.setApplication("odc"); + entity.setProfile("default"); entity.setDescription("description"); return entity; } diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java index 2db6c9bffc..1d20c6ada8 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java @@ -39,6 +39,8 @@ public class JobRepositoryTest extends ServiceTestEnv { public void test_saveJob() { JobEntity je = createJobEntity(); Assert.assertNotNull(je); + jobRepository.delete(je); + Assert.assertFalse(jobRepository.existsById(je.getId())); } @Test @@ -53,6 +55,8 @@ public void updateReportResult() { jse.setFinishedTime(JobDateUtils.getCurrentDate()); int rows = jobRepository.updateReportResult(jse, currentJob.getId(), currentJob.getStatus()); Assert.assertTrue(rows > 0); + jobRepository.deleteById(currentJob.getId()); + Assert.assertFalse(jobRepository.existsById(currentJob.getId())); } private JobEntity createJobEntity() { diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java index d2d3e14418..f44e2e90c9 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java @@ -70,6 +70,7 @@ import com.oceanbase.odc.service.flow.model.FlowTaskExecutionStrategy; import com.oceanbase.odc.service.flow.tool.TestFlowRuntimeTaskImpl; import com.oceanbase.odc.service.flow.util.FlowInstanceUtil; +import com.oceanbase.odc.service.iam.UserService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.test.tool.TestRandom; @@ -115,6 +116,8 @@ public class FlowResponseMapperFactoryTest extends ServiceTestEnv { private GateWayInstanceRepository gateWayInstanceRepository; @MockBean private DatabaseService databaseService; + @MockBean + private UserService userService; @Autowired private UserTaskInstanceCandidateRepository userTaskInstanceCandidateRepository; private final AtomicLong counter = new AtomicLong(0L); @@ -162,6 +165,9 @@ public void generateFlowInstanceDetailResp_instanceInput_returnDesp() { List flowInstanceIds = Collections.singletonList(flowInstance.getId()); when(databaseService.listDatabasesByConnectionIds(Mockito.anyCollection())) .thenReturn(Collections.singletonList(getDatabase())); + UserEntity userEntity = new UserEntity(); + userEntity.setEnabled(true); + when(userService.nullSafeGet(1L)).thenReturn(userEntity); FlowInstanceMapper mapper = responseFactory.generateMapperByInstanceIds(flowInstanceIds); FlowNodeInstanceMapper nodeMapper = responseFactory.generateNodeMapperByInstanceIds(flowInstanceIds); FlowInstanceDetailResp resp = mapper.map(flowInstance, nodeMapper); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeExpiredTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeExpiredTest.java index d135a70bb7..26ef9ca9e1 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeExpiredTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeExpiredTest.java @@ -56,7 +56,7 @@ public void test_osc_task_expired_after_seconds() { onlineSchemaChangeTaskHandler.complete(schedule.getId(), taskEntities.get(0).getId()); Assert.assertEquals(2, taskEntities.size()); - Assert.assertEquals(TaskStatus.CANCELED, + Assert.assertEquals(TaskStatus.FAILED, scheduleTaskRepository.findById(taskEntities.get(0).getId()).get().getStatus()); Assert.assertEquals(TaskStatus.PREPARING, scheduleTaskRepository.findById(taskEntities.get(1).getId()).get().getStatus()); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/resultset/DumperResultSetExportTaskManagerTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/resultset/DumperResultSetExportTaskManagerTest.java index 2ff121281b..a9c7fb2d9d 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/resultset/DumperResultSetExportTaskManagerTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/resultset/DumperResultSetExportTaskManagerTest.java @@ -26,9 +26,8 @@ import org.apache.commons.io.FileUtils; import org.junit.After; -import org.junit.AfterClass; import org.junit.Assert; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -43,12 +42,13 @@ import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.flow.task.model.ResultSetExportResult; import com.oceanbase.odc.service.resultset.ResultSetExportTaskParameter.CSVFormat; +import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; public class DumperResultSetExportTaskManagerTest extends ServiceTestEnv { - private static ConnectionSession mysqlSession; - private static ConnectionSession oracleSession; - private static ConnectionConfig mysqlConnecitonConfig; - private static ConnectionConfig oracleConnecitonConfig; + private ConnectionSession mysqlSession; + private ConnectionSession oracleSession; + private ConnectionConfig mysqlConnectionConfig; + private ConnectionConfig oracleConnectionConfig; private final String userId = "1"; private final String taskId = "1"; private final String fileName = "CUSTOM_SQL"; @@ -59,12 +59,12 @@ public class DumperResultSetExportTaskManagerTest extends ServiceTestEnv { @Autowired private DumperResultSetExportTaskManager manager; - @BeforeClass - public static void classUp() { - mysqlSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); - mysqlConnecitonConfig = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(mysqlSession); - oracleSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_ORACLE); - oracleConnecitonConfig = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(oracleSession); + @Before + public void init() { + mysqlSession = getRowTestConnectionSession(ConnectType.OB_MYSQL); + mysqlConnectionConfig = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(mysqlSession); + oracleSession = getRowTestConnectionSession(ConnectType.OB_ORACLE); + oracleConnectionConfig = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(oracleSession); String sql = "create table rs_export_test(`col1` varchar(64), `col2` varchar(64), `col3` varchar(64), `col4` varchar(64))"; mysqlSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY).update(sql); @@ -83,24 +83,31 @@ public static void classUp() { oracleDefaultSchema = ConnectionSessionUtil.getCurrentSchema(oracleSession); } - @AfterClass - public static void classDown() { + private ConnectionSession getRowTestConnectionSession(ConnectType connectType) { + ConnectionConfig config = TestConnectionUtil.getTestConnectionConfig(connectType); + return new DefaultConnectSessionFactory(config).generateSession(); + } + + public void releaseResource() { String sql = "drop table rs_export_test"; mysqlSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY).update(sql); oracleSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY).update(sql); + mysqlSession.expire(); + oracleSession.expire(); } @After public void tearDown() { File workingDir = Paths.get(basePath, taskId).toFile(); FileUtils.deleteQuietly(workingDir); + releaseResource(); } @Test public void startTask_ExportSQL_GenerateFileSuccess() throws Exception { ResultSetExportTaskParameter req = createResultSetExportTaskReq(DataTransferFormat.SQL, EncodingType.UTF_8, mysqlSession); - ResultSetExportTaskContext context = manager.start(mysqlConnecitonConfig, req, taskId); + ResultSetExportTaskContext context = manager.start(mysqlConnectionConfig, req, taskId); await().atMost(30, SECONDS).until(context::isDone); ResultSetExportResult result = context.get(); File file = Paths.get(basePath, taskId, fileName + req.getFileFormat().getExtension()).toFile(); @@ -115,7 +122,7 @@ public void startTask_ExportCSV_GenerateFileSuccess() throws Exception { CSVFormat csvFormat = new CSVFormat(); csvFormat.setColumnDelimiter('"'); req.setCsvFormat(csvFormat); - ResultSetExportTaskContext context = manager.start(mysqlConnecitonConfig, req, taskId); + ResultSetExportTaskContext context = manager.start(mysqlConnectionConfig, req, taskId); await().atMost(30, SECONDS).until(context::isDone); File file = Paths.get(basePath, taskId, fileName + req.getFileFormat().getExtension()).toFile(); LineNumberReader lineNumberReader = new LineNumberReader(new FileReader(file)); @@ -136,7 +143,7 @@ public void startTask_ExportCSV_ExportFailed() { csvFormat.setColumnDelimiter('"'); req.setCsvFormat(csvFormat); try { - ResultSetExportTaskContext context = manager.start(mysqlConnecitonConfig, req, taskId); + ResultSetExportTaskContext context = manager.start(mysqlConnectionConfig, req, taskId); await().atMost(30, SECONDS).until(context::isDone); context.get(); } catch (Exception e) { @@ -149,7 +156,7 @@ public void startTask_ExportEXCEL_GenerateFileSuccess() throws Exception { ResultSetExportTaskParameter req = createResultSetExportTaskReq(DataTransferFormat.EXCEL, EncodingType.GBK, mysqlSession); req.setCsvFormat(new CSVFormat()); - ResultSetExportTaskContext context = manager.start(mysqlConnecitonConfig, req, taskId); + ResultSetExportTaskContext context = manager.start(mysqlConnectionConfig, req, taskId); await().atMost(30, SECONDS).until(context::isDone); File file = Paths.get(basePath, taskId, fileName + req.getFileFormat().getExtension()).toFile(); Assert.assertTrue(file.exists()); @@ -161,7 +168,7 @@ public void startTask_ExportSQL_GenerateFileSuccess_WithoutTableName() throws Ex ResultSetExportTaskParameter req = createResultSetExportTaskReq(DataTransferFormat.SQL, EncodingType.UTF_8, mysqlSession); req.setTableName(null); - ResultSetExportTaskContext context = manager.start(mysqlConnecitonConfig, req, taskId); + ResultSetExportTaskContext context = manager.start(mysqlConnectionConfig, req, taskId); await().atMost(30, SECONDS).until(context::isDone); File file = Paths.get(basePath, taskId, fileName + req.getFileFormat().getExtension()).toFile(); Assert.assertTrue(file.exists()); @@ -175,7 +182,7 @@ public void startTask_ExportCSV_Oracle_GenerateFileSuccess() throws Exception { CSVFormat csvFormat = new CSVFormat(); csvFormat.setColumnDelimiter('"'); req.setCsvFormat(csvFormat); - ResultSetExportTaskContext context = manager.start(oracleConnecitonConfig, req, taskId); + ResultSetExportTaskContext context = manager.start(oracleConnectionConfig, req, taskId); await().atMost(15, SECONDS).until(context::isDone); File file = Paths.get(basePath, taskId, fileName + req.getFileFormat().getExtension()).toFile(); LineNumberReader lineNumberReader = new LineNumberReader(new FileReader(file)); @@ -192,7 +199,7 @@ public void startTask_ExportCSV_WithMaxRowsLimit_MySQL() throws Exception { CSVFormat csvFormat = new CSVFormat(); csvFormat.setColumnDelimiter('"'); req.setCsvFormat(csvFormat); - ResultSetExportTaskContext context = manager.start(oracleConnecitonConfig, req, taskId); + ResultSetExportTaskContext context = manager.start(oracleConnectionConfig, req, taskId); await().atMost(15, SECONDS).until(context::isDone); File file = Paths.get(basePath, taskId, fileName + req.getFileFormat().getExtension()).toFile(); LineNumberReader lineNumberReader = new LineNumberReader(new FileReader(file)); @@ -209,7 +216,7 @@ public void startTask_ExportCSV_WithMaxRowsLimit_Oracle() throws Exception { CSVFormat csvFormat = new CSVFormat(); csvFormat.setColumnDelimiter('"'); req.setCsvFormat(csvFormat); - ResultSetExportTaskContext context = manager.start(oracleConnecitonConfig, req, taskId); + ResultSetExportTaskContext context = manager.start(oracleConnectionConfig, req, taskId); await().atMost(15, SECONDS).until(context::isDone); File file = Paths.get(basePath, taskId, fileName + req.getFileFormat().getExtension()).toFile(); LineNumberReader lineNumberReader = new LineNumberReader(new FileReader(file)); diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/json/JsonUtils.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/json/JsonUtils.java index 47185c1ed6..b26258bd3e 100644 --- a/server/odc-common/src/main/java/com/oceanbase/odc/common/json/JsonUtils.java +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/json/JsonUtils.java @@ -72,6 +72,7 @@ public static T fromJson(String json, Class classType) { try { return OBJECT_MAPPER.readValue(json, classType); } catch (JsonProcessingException e) { + log.warn("deserialize str = {} failed", json, e); return null; } } @@ -94,6 +95,7 @@ public static T fromJson(String json, JavaType javaType) { try { return OBJECT_MAPPER.readValue(json, javaType); } catch (JsonProcessingException e) { + log.warn("deserialize str = {} failed", json, e); return null; } } @@ -109,6 +111,7 @@ public static T fromJson(String json, TypeReference valueTypeRef) { try { return OBJECT_MAPPER.readValue(json, valueTypeRef); } catch (JsonProcessingException e) { + log.warn("deserialize str = {} failed", json, e); return null; } } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java index d1961279f1..8469af1795 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java @@ -54,6 +54,10 @@ public enum FlowStatus { * {@code FlowInstance} is succeed */ EXECUTION_SUCCEEDED, + /** + * {@code FlowInstance} is failed + */ + EXECUTION_ABNORMAL, /** * {@code FlowInstance} is failed */ diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java index aea0c1842a..c2290e57e2 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java @@ -25,6 +25,9 @@ public enum TaskStatus { PREPARING, RUNNING, + // task not work, but can be recovered + ABNORMAL, + // the following is terminate states FAILED, CANCELED, DONE; diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/OscController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/OscController.java index 7609ce7f65..3b25740644 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/OscController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/OscController.java @@ -63,4 +63,11 @@ public SuccessResponse updateRateLimitConfig( return Responses.success(oscService.updateRateLimiterConfig(updateRateLimiterConfig)); } + @ApiOperation(value = "oscResume", notes = "resume failed osc task") + @RequestMapping(value = "/{flowInstanceId:[\\d]+}/resume", method = RequestMethod.POST) + public SuccessResponse resumeOscTask( + @PathVariable Long flowInstanceId) { + return Responses.success(oscService.resumeOscTask(flowInstanceId)); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/BeanInjectedClassDelegate.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/BeanInjectedClassDelegate.java index 39cd4986f7..b21401b98c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/BeanInjectedClassDelegate.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/BeanInjectedClassDelegate.java @@ -70,6 +70,11 @@ protected Object instantiateDelegate(String className, List fi return obj; } + /** + * this function will create Delegate class and call post Construct method eg: for + * {@link com.oceanbase.odc.service.flow.task.BaseRuntimeFlowableDelegate}'s post construct will add + * new flow instance listener to LocalEvent publisher + */ public static T instantiateDelegate(Class beanClass) throws Exception { T beanInstance = beanClass.getDeclaredConstructor().newInstance(); forEachClass(beanClass, c -> injectAutowiredBeans(beanInstance, c)); @@ -77,6 +82,17 @@ public static T instantiateDelegate(Class beanClass) throws Exception { return beanInstance; } + /** + * use this function to avoid event publisher register and unregister if use instantiateDelegate and + * unregister listener not called, flow create instance listener will be add to LocalEventPublisher. + * That will cause memory leak and unexpected result + */ + public static T instantiateDelegateWithoutPostConstructInvoke(Class beanClass) throws Exception { + T beanInstance = beanClass.getDeclaredConstructor().newInstance(); + forEachClass(beanClass, c -> injectAutowiredBeans(beanInstance, c)); + return beanInstance; + } + private static void forEachClass(@NonNull Class begin, Consumer> consumer) { Stack> stack = new Stack<>(); stack.push(begin); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java index 5944bbcf49..488ec99c5b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java @@ -657,14 +657,23 @@ private FlowInstanceDetailResp cancel(@NotNull FlowInstance flowInstance, Boolea } } if (CollectionUtils.isEmpty(approvalInstances) && CollectionUtils.isEmpty(taskInstances)) { + // throw new UnsupportedException(ErrorCodes.FinishedTaskNotTerminable, null, + // "The current task has been completed and cannot be terminated"); + log.warn("The current task has been completed, flowInstance = {}", flowInstance); + } + // check state before do update + if (FlowStatus.CANCELLED != flowInstance.getStatus() && FlowStatus.COMPLETED != flowInstance.getStatus()) { + log.info("Flow status error, cancellation conditions are not met, forced cancellation, flowInstanceId={}, " + + "nodes={}", id, + instances.stream().map(t -> t.getId() + "," + t.getNodeType() + "," + t.getStatus()) + .collect(Collectors.toList())); + flowInstanceRepository.updateStatusById(id, FlowStatus.CANCELLED); + } else { + log.warn("Flow status error, unexpected flow stats for cancellation, flowInstanceId={}, current status={}", + id, flowInstance.getId()); throw new UnsupportedException(ErrorCodes.FinishedTaskNotTerminable, null, "The current task has been completed and cannot be terminated"); } - log.info("Flow status error, cancellation conditions are not met, forced cancellation, flowInstanceId={}, " - + "nodes={}", id, - instances.stream().map(t -> t.getId() + "," + t.getNodeType() + "," + t.getStatus()) - .collect(Collectors.toList())); - flowInstanceRepository.updateStatusById(id, FlowStatus.CANCELLED); return FlowInstanceDetailResp.withIdAndType(id, taskTypeHolder.getValue()); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseODCFlowTaskDelegate.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseODCFlowTaskDelegate.java index b8779086a5..b9186141da 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseODCFlowTaskDelegate.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseODCFlowTaskDelegate.java @@ -84,7 +84,7 @@ public abstract class BaseODCFlowTaskDelegate extends BaseRuntimeFlowableDele protected ServiceTaskInstanceRepository serviceTaskRepository; private final CountDownLatch taskLatch = new CountDownLatch(1); @Getter - private volatile Long taskId; + protected volatile Long taskId; @Getter private volatile long timeoutMillis; @Getter @@ -101,14 +101,14 @@ public abstract class BaseODCFlowTaskDelegate extends BaseRuntimeFlowableDele @Autowired private FlowTaskInstanceService flowTaskInstanceService; - private void init(DelegateExecution execution) { + protected void init(DelegateExecution execution) { this.taskId = FlowTaskUtil.getTaskId(execution); this.timeoutMillis = getTimeoutMillis(execution); this.taskService.updateExecutorInfo(taskId, new ExecutorInfo(hostProperties)); SecurityContextUtils.setCurrentUser(FlowTaskUtil.getTaskCreator(execution)); } - private void initMonitorExecutor() { + protected void initMonitorExecutor() { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("Task-Periodically-Scheduled-%d") .build(); @@ -178,6 +178,13 @@ protected void run(DelegateExecution execution) throws Exception { SecurityContextUtils.clear(); throw new ServiceTaskError(e); } + completeTask(); + } + + /** + * wait task finish and do the remain job + */ + protected void completeTask() { try { taskLatch.await(timeoutMillis, TimeUnit.MILLISECONDS); if (isCancelled()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseRuntimeFlowableDelegate.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseRuntimeFlowableDelegate.java index 47796784a9..a42ca2ff0c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseRuntimeFlowableDelegate.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/BaseRuntimeFlowableDelegate.java @@ -75,7 +75,7 @@ public abstract class BaseRuntimeFlowableDelegate extends BaseFlowableDelegat @Getter private TaskType taskType; @Getter - private Long flowInstanceId; + protected Long flowInstanceId; @Getter private ExecutionStrategyConfig strategyConfig; @Autowired diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java index f4a5a5face..7b9d314178 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java @@ -16,10 +16,8 @@ package com.oceanbase.odc.service.onlineschemachange; import java.util.Date; -import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; @@ -27,13 +25,17 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; +import com.google.common.annotations.VisibleForTesting; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.constant.FlowStatus; import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.core.shared.constant.TaskType; import com.oceanbase.odc.metadb.schedule.ScheduleEntity; import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskSpecs; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; @@ -51,10 +53,8 @@ import com.oceanbase.odc.service.quartz.QuartzJobService; import com.oceanbase.odc.service.quartz.model.MisfireStrategy; import com.oceanbase.odc.service.schedule.ScheduleService; -import com.oceanbase.odc.service.schedule.ScheduleTaskService; import com.oceanbase.odc.service.schedule.model.JobType; import com.oceanbase.odc.service.schedule.model.ScheduleStatus; -import com.oceanbase.odc.service.schedule.model.ScheduleTask; import com.oceanbase.odc.service.schedule.model.ScheduleType; import com.oceanbase.odc.service.schedule.model.TriggerConfig; import com.oceanbase.odc.service.schedule.model.TriggerStrategy; @@ -76,8 +76,6 @@ public class OnlineSchemaChangeFlowableTask extends BaseODCFlowTaskDelegate lastManualSwapTableEnableTasks = new HashSet<>(); + private TaskStatus taskStatus; @Override protected Void start(Long taskId, TaskService taskService, DelegateExecution execution) throws Exception { @@ -103,15 +98,14 @@ protected Void start(Long taskId, TaskService taskService, DelegateExecution exe User creator = FlowTaskUtil.getTaskCreator(execution); this.creatorId = creator.getId(); this.organizationId = creator.getOrganizationId(); - this.status = TaskStatus.RUNNING; long flowTaskId = taskId; // for public cloud String uid = FlowTaskUtil.getCloudMainAccountId(execution); OnlineSchemaChangeParameters parameter = FlowTaskUtil.getOnlineSchemaChangeParameter(execution); parameter.setFlowInstanceId(FlowTaskUtil.getFlowInstanceId(execution)); + parameter.setFlowTaskID(taskId); ConnectionConfig connectionConfig = FlowTaskUtil.getConnectionConfig(execution); String schema = FlowTaskUtil.getSchemaName(execution); - continueOnError = parameter.isContinueOnError(); OnlineSchemaChangeContextHolder.trace(this.creatorId, flowTaskId, this.organizationId); TaskEntity taskEntity = taskService.detail(taskId); Database database = databaseService.getBasicSkipPermissionCheck(taskEntity.getDatabaseId()); @@ -134,85 +128,145 @@ protected Void start(Long taskId, TaskService taskService, DelegateExecution exe } finally { OnlineSchemaChangeContextHolder.clear(); } + updateFlowInstanceStatus(FlowStatus.EXECUTING); return null; } @Override - protected boolean isSuccessful() { - return status == TaskStatus.DONE; - } - - @Override - protected boolean isFailure() { - return status == TaskStatus.FAILED; - } - - @Override - protected void onFailure(Long taskId, TaskService taskService) { + public void onFailure(Long taskId, TaskService taskService) { log.warn("Online schema change task failed, taskId={}", taskId); super.onFailure(taskId, taskService); + updateFlowInstanceStatus(FlowStatus.EXECUTION_FAILED); } @Override - protected void onSuccessful(Long taskId, TaskService taskService) { + public void onSuccessful(Long taskId, TaskService taskService) { log.info("Online schema change task succeed, taskId={}", taskId); super.onSuccessful(taskId, taskService); updateFlowInstanceStatus(FlowStatus.EXECUTION_SUCCEEDED); } @Override - protected void onTimeout(Long taskId, TaskService taskService) { + public void onTimeout(Long taskId, TaskService taskService) { log.warn("Online schema change task timeout, taskId={}", taskId); super.onTimeout(taskId, taskService); + updateFlowInstanceStatus(FlowStatus.EXECUTION_EXPIRED); } + /** + * refresh task's state and try complete task + * + * @param flowInstanceID instance id of flow_instance + * @param taskID task id of task_task + * @param schedulerID scheduler id of scheduler_scheduler + */ + public void tryCompleteTask(long flowInstanceID, long taskID, long schedulerID) { + this.flowInstanceId = flowInstanceID; + this.taskId = taskID; + this.scheduleId = schedulerID; + onProgressUpdate(taskID, taskService); + switch (taskStatus) { + case ABNORMAL: + processAbnormal(taskID, taskService); + break; + case FAILED: + onFailure(taskID, taskService); + break; + case CANCELED: + cancel(true, taskID, taskService); + break; + case DONE: + onSuccessful(taskID, taskService); + break; + default: // do nothing + } + } + + /** + * update progress when action changed + * + * @param taskId taskID for task_task table + * @param taskService + */ @Override - protected void onProgressUpdate(Long taskId, TaskService taskService) { + public void onProgressUpdate(Long taskId, TaskService taskService) { try { - Page tasks = scheduleTaskService.listEntity(Pageable.unpaged(), scheduleId); - if (tasks.getSize() == 0) { + Page scheduleTasks = list(Pageable.unpaged(), scheduleId); + if (scheduleTasks.getSize() == 0) { log.info("List schedule task size is 0 by scheduleId {}.", scheduleId); return; } - progressStatusUpdate(tasks); - - Optional res = tasks.stream().map(this::singleTaskPercentage).reduce(Double::sum); - double currentPercentage = res.get() * 100 / tasks.getSize(); - + // compute status + TaskStatus currentStatus = computeStatus(scheduleTasks); + this.taskStatus = currentStatus; TaskEntity flowTask = taskService.detail(taskId); TaskStatus dbStatus = flowTask.getStatus(); + OnlineSchemaChangeTaskResult previousTaskResult = + JsonUtils.fromJson(flowTask.getResultJson(), OnlineSchemaChangeTaskResult.class); + // get prev and current + double currentProgressPercentage = getProgressPercentage(scheduleTasks); + double prevProgressPercentage = + null != previousTaskResult ? getProgressPercentage(previousTaskResult.getTasks()) : 0; + // get swap table flag changed + int currentEnableManualSwapTableFlagCounts = getManualSwapTableEnableFlagCounts(scheduleTasks); + int prevEnableManualSwapTableFlagCounts = + null != previousTaskResult ? getManualSwapTableEnableFlagCounts(previousTaskResult.getTasks()) : 0; - Set currentManualSwapTableEnableTasks = new HashSet<>(); - for (ScheduleTaskEntity task : tasks) { - OnlineSchemaChangeScheduleTaskResult result = JsonUtils.fromJson(task.getResultJson(), - OnlineSchemaChangeScheduleTaskResult.class); - if (result.isManualSwapTableEnabled()) { - currentManualSwapTableEnableTasks.add(task.getId()); - } - } - - boolean manualSwapTableTasksChanged = !CollectionUtils.isEqualCollection(currentManualSwapTableEnableTasks, - lastManualSwapTableEnableTasks); - if (manualSwapTableTasksChanged) { - lastManualSwapTableEnableTasks = currentManualSwapTableEnableTasks; - } - - if (currentPercentage > this.percentage || dbStatus != this.status || manualSwapTableTasksChanged) { - flowTask.setResultJson(JsonUtils.toJson(new OnlineSchemaChangeTaskResult(tasks.getContent()))); - flowTask.setStatus(this.status); - flowTask.setProgressPercentage(Math.min(currentPercentage, 100)); + if (currentProgressPercentage > prevProgressPercentage || dbStatus != currentStatus + || (currentEnableManualSwapTableFlagCounts != prevEnableManualSwapTableFlagCounts)) { + flowTask.setResultJson(JsonUtils.toJson(new OnlineSchemaChangeTaskResult(scheduleTasks.getContent()))); + flowTask.setStatus(currentStatus); + flowTask.setProgressPercentage(Math.min(currentProgressPercentage, 100)); taskService.update(flowTask); } - this.percentage = currentPercentage; } catch (Throwable ex) { log.warn("onProgressUpdate occur exception,", ex); } + } + + /** + * get counts of tasks with swap table flag enabled + */ + protected int getManualSwapTableEnableFlagCounts(Iterable tasks) { + if (null == tasks) { + return 0; + } + int ret = 0; + for (ScheduleTaskEntity task : tasks) { + // check current + OnlineSchemaChangeScheduleTaskResult result = JsonUtils.fromJson(task.getResultJson(), + OnlineSchemaChangeScheduleTaskResult.class); + if (null != result && result.isManualSwapTableEnabled()) { + ret++; + } + } + return ret; + } + protected double getProgressPercentage(Iterable tasks) { + if (null == tasks) { + return 0.0; + } + int taskCounts = 0; + double percentageSum = 0.0; + for (ScheduleTaskEntity task : tasks) { + percentageSum += singleTaskPercentage(task); + taskCounts++; + } + return taskCounts == 0 ? 0 : percentageSum * 100 / taskCounts; } - private void progressStatusUpdate(Page tasks) { + /** + * compute current status by saved task status + * + * @param tasks + * @return + */ + @VisibleForTesting + protected TaskStatus computeStatus(Page tasks) { int successfulTask = 0; int failedTask = 0; + int abnormalTask = 0; boolean canceled = false; for (ScheduleTaskEntity task : tasks) { @@ -223,15 +277,21 @@ private void progressStatusUpdate(Page tasks) { failedTask++; } else if (taskStatus == TaskStatus.CANCELED) { canceled = true; + } else if (taskStatus == TaskStatus.ABNORMAL) { + abnormalTask++; } } if (canceled) { - this.status = TaskStatus.CANCELED; + return TaskStatus.CANCELED; + } else if (abnormalTask != 0) { + return TaskStatus.ABNORMAL; } else if ((successfulTask + failedTask) == tasks.getSize()) { - this.status = failedTask == 0 ? TaskStatus.DONE : TaskStatus.FAILED; - } else if (failedTask > 0 && !continueOnError) { - this.status = TaskStatus.FAILED; + return failedTask == 0 ? TaskStatus.DONE : TaskStatus.FAILED; + } else if (failedTask > 0) { + return TaskStatus.FAILED; + } else { + return TaskStatus.RUNNING; } } @@ -242,6 +302,7 @@ private double singleTaskPercentage(ScheduleTaskEntity scheduleTask) { percentage = 0; break; case RUNNING: + case ABNORMAL: percentage = scheduleTask.getProgressPercentage() / 100; break; default: @@ -252,19 +313,34 @@ private double singleTaskPercentage(ScheduleTaskEntity scheduleTask) { } @Override - protected boolean cancel(boolean mayInterruptIfRunning, Long taskId, TaskService taskService) { - Optional runningEntity = scheduleTaskService.list(Pageable.unpaged(), scheduleId) - .stream().filter(task -> task.getStatus() == TaskStatus.RUNNING) - .findFirst(); + public boolean cancel(boolean mayInterruptIfRunning, Long taskId, TaskService taskService) { + Page scheduleTasks = list(Pageable.unpaged(), scheduleId); + Optional runningEntity = Optional.empty(); + for (ScheduleTaskEntity task : scheduleTasks) { + if (TaskStatus.RUNNING == task.getStatus()) { + runningEntity = Optional.of(task); + break; + } + } runningEntity.ifPresent(scheduleTask -> taskHandler.terminate(scheduleId, scheduleTask.getId())); - this.status = TaskStatus.CANCELED; taskService.cancel(taskId); return true; } - @Override - public boolean isCancelled() { - return status == TaskStatus.CANCELED; + public void processAbnormal(Long taskId, TaskService taskService) { + log.info("Online schema change task abnormal, taskId={}", taskId); + updateFlowInstanceStatus(FlowStatus.EXECUTION_ABNORMAL); + } + + /** + * query all scheduler task entity with scheduleId + * + * @return + */ + protected Page list(Pageable pageable, Long scheduleId) { + Specification specification = + Specification.where(ScheduleTaskSpecs.jobNameEquals(scheduleId.toString())); + return scheduleTaskRepository.findAll(specification, pageable); } private ScheduleEntity createScheduleEntity(ConnectionConfig connectionConfig, @@ -302,4 +378,28 @@ private ScheduleTaskEntity createScheduleTaskEntity(Long scheduleId, return scheduleTaskRepository.saveAndFlush(scheduleTaskEntity); } + public TaskType getTaskType() { + return TaskType.ONLINE_SCHEMA_CHANGE; + } + + // do nothing + protected void initMonitorExecutor() {} + + // do nothing + protected void completeTask() {} + + @Override + public boolean isCancelled() { + throw new RuntimeException("not impl"); + } + + @Override + protected boolean isSuccessful() { + throw new RuntimeException("not impl"); + } + + @Override + protected boolean isFailure() { + throw new RuntimeException("not impl"); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java index c4ab8c8ef3..a0f842d05f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java @@ -16,8 +16,15 @@ package com.oceanbase.odc.service.onlineschemachange; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; + +import javax.validation.constraints.NotNull; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -33,10 +40,13 @@ import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.shared.constant.ErrorCodes; +import com.oceanbase.odc.core.shared.constant.FlowStatus; import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.core.shared.constant.TaskType; import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.metadb.connection.DatabaseRepository; +import com.oceanbase.odc.metadb.flow.FlowInstanceRepository; import com.oceanbase.odc.metadb.schedule.ScheduleEntity; import com.oceanbase.odc.metadb.schedule.ScheduleRepository; import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; @@ -59,6 +69,7 @@ import com.oceanbase.odc.service.onlineschemachange.model.RateLimiterConfig; import com.oceanbase.odc.service.onlineschemachange.model.SwapTableType; import com.oceanbase.odc.service.onlineschemachange.model.UpdateRateLimiterConfigRequest; +import com.oceanbase.odc.service.onlineschemachange.oscfms.ActionScheduler; import com.oceanbase.odc.service.onlineschemachange.rename.OscDBUserUtil; import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.service.schedule.ScheduleTaskService; @@ -97,11 +108,15 @@ public class OscService { @Autowired private FlowInstanceService flowInstanceService; @Autowired + private FlowInstanceRepository flowInstanceRepository; + @Autowired private ScheduleTaskService scheduleTaskService; @Autowired private ScheduleService scheduleService; @Autowired private TaskService taskService; + @Autowired + private ActionScheduler actionScheduler; @SkipAuthorize("internal authenticated") @@ -207,6 +222,69 @@ public boolean updateRateLimiterConfig(UpdateRateLimiterConfigRequest req) { return true; } + /** + * resume osc task 1. get task_task result 2. reset flow / task / scheduler_task status by + * resetTaskStatus flag 3. restart scheduler task + * + * @param flowInstanceID flow instance id + * @return + */ + @Transactional(rollbackFor = Exception.class) + @SkipAuthorize("internal authenticated") + public boolean resumeOscTask(@NotNull Long flowInstanceID) { + log.info("resume osc task for flowInstanceID = {}", flowInstanceID); + // check permission + checkPermission(flowInstanceID); + // verify flow instance status + Map flowInstanceStatus = flowInstanceService.getStatus(Collections.singleton(flowInstanceID)); + FlowStatus flowStatus = flowInstanceStatus.get(flowInstanceID); + if (FlowStatus.EXECUTING == flowStatus) { + log.info("task is in execution status, resume ignored, flowInstanceID = {}", flowInstanceID); + } + PreConditions.validArgumentState(FlowStatus.EXECUTION_ABNORMAL == flowStatus, ErrorCodes.Unsupported, + new Object[] {flowInstanceID}, + "only status in ABNORMAL state can resume, flowInstanceID=" + flowInstanceID); + // get task_task and result json + TaskEntity task = flowInstanceService.getTaskByFlowInstanceId(flowInstanceID); + PreConditions.notNull(task.getResultJson(), "result json", + "Task result is empty, taskId=" + task.getId() + ",flowInstanceId=" + flowInstanceID); + OnlineSchemaChangeTaskResult taskResult = + JsonUtils.fromJson(task.getResultJson(), new TypeReference() {}); + // find the failed schedule task, change it to running + ScheduleTaskEntity abnormalTask = findAbnormalTask(task, flowInstanceID, taskResult); + // update flow instance, change it to execution + flowInstanceRepository.updateStatusById(flowInstanceID, FlowStatus.EXECUTING); + // update task task, change it to running + task.setStatus(TaskStatus.RUNNING); + taskService.update(task); + scheduleTaskService.updateStatusById(abnormalTask.getId(), TaskStatus.RUNNING); + // try register schedule with task id. + Long scheduleId = Long.parseLong(taskResult.getTasks().get(0).getJobName()); + ScheduleEntity scheduleEntity = scheduleService.nullSafeGetById(scheduleId); + actionScheduler.submitFMSScheduler(scheduleEntity, abnormalTask.getId()); + return true; + } + + private ScheduleTaskEntity findAbnormalTask(TaskEntity task, Long flowInstanceID, + OnlineSchemaChangeTaskResult onlineSchemaChangeTaskResult) { + List failedTasks = new ArrayList<>(); + PreConditions.validArgumentState(CollectionUtils.isNotEmpty(onlineSchemaChangeTaskResult.getTasks()), + ErrorCodes.IllegalArgument, + new Object[] {flowInstanceID}, + "Task result is empty, taskId=" + task.getId() + ",flowInstanceId=" + flowInstanceID); + for (ScheduleTaskEntity scheduleTask : onlineSchemaChangeTaskResult.getTasks()) { + if (scheduleTask.getStatus() == TaskStatus.ABNORMAL) { + failedTasks.add(scheduleTask); + } + } + PreConditions.validArgumentState(failedTasks.size() == 1, ErrorCodes.IllegalArgument, + new Object[] {flowInstanceID}, + "Failed schedule task count not valid, taskId=" + task.getId() + ",flowInstanceId=" + flowInstanceID + + ", scheduler task id = " + + failedTasks.stream().map(ScheduleTaskEntity::getId).collect(Collectors.toList())); + return failedTasks.get(0); + } + private void checkPermission(Long flowInstanceId) { Optional optional = flowFactory.getFlowInstance(flowInstanceId); FlowInstance flowInstance = optional.orElseThrow( @@ -245,6 +323,4 @@ private boolean getLockUserIsRequired(Long connectionId) { return version; }); } - - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeParameters.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeParameters.java index 338cf74fa5..d65449d7cb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeParameters.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/model/OnlineSchemaChangeParameters.java @@ -63,6 +63,8 @@ public class OnlineSchemaChangeParameters implements Serializable, TaskParameter private OnlineSchemaChangeSqlType sqlType; private String sqlContent; + // resume has be introduced to osc task, so error strategy is not necessary + @Deprecated private TaskErrorStrategy errorStrategy; private Integer lockTableTimeOutSeconds; @@ -75,7 +77,10 @@ public class OnlineSchemaChangeParameters implements Serializable, TaskParameter private List lockUsers; private SwapTableType swapTableType; + // flow instance id private Long flowInstanceId; + // flow task id + private Long flowTaskID; private RateLimiterConfig rateLimitConfig = new RateLimiterConfig(); public boolean isContinueOnError() { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/OmsRequestUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/OmsRequestUtil.java deleted file mode 100644 index b3439cfa2f..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/OmsRequestUtil.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.onlineschemachange.oms; - -import java.util.List; -import java.util.Map; - -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.core.shared.constant.TaskStatus; -import com.oceanbase.odc.service.onlineschemachange.configuration.OnlineSchemaChangeProperties; -import com.oceanbase.odc.service.onlineschemachange.model.FullVerificationResult; -import com.oceanbase.odc.service.onlineschemachange.oms.enums.OmsStepName; -import com.oceanbase.odc.service.onlineschemachange.oms.openapi.OmsProjectOpenApiService; -import com.oceanbase.odc.service.onlineschemachange.oms.request.ListOmsProjectFullVerifyResultRequest; -import com.oceanbase.odc.service.onlineschemachange.oms.request.OmsProjectControlRequest; -import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectFullVerifyResultResponse; -import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectProgressResponse; -import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectStepVO; -import com.oceanbase.odc.service.onlineschemachange.oscfms.action.oms.ProjectStepResultChecker; -import com.oceanbase.odc.service.onlineschemachange.oscfms.action.oms.ProjectStepResultChecker.ProjectStepResult; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author longpeng.zlp - * @date 2024/7/29 14:01 - * @since 4.3.1 - */ -@Slf4j -public class OmsRequestUtil { - /** - * - * @return - */ - public static OmsProjectControlRequest getProjectRequest(String uid, String projectID) { - OmsProjectControlRequest projectRequest = new OmsProjectControlRequest(); - projectRequest.setUid(uid); - projectRequest.setId(projectID); - return projectRequest; - } - - public static boolean isOmsTaskReady(ProjectStepResult projectStepResult) { - return projectStepResult.getTaskStatus() == TaskStatus.DONE - && (projectStepResult.getFullVerificationResult() == FullVerificationResult.CONSISTENT || - projectStepResult.getFullVerificationResult() == FullVerificationResult.UNCHECK); - } - - public static ProjectStepResult buildProjectStepResult(OmsProjectOpenApiService omsProjectOpenApiService, - OnlineSchemaChangeProperties onlineSchemaChangeProperties, - String uid, String projectID, String databaseName, - Map checkFailedTimes) { - // do remain job - OmsProjectControlRequest projectRequest = getProjectRequest(uid, projectID); - List projectSteps = omsProjectOpenApiService.describeProjectSteps(projectRequest); - if (log.isDebugEnabled()) { - log.debug("Get project step list from projectOpenApiService is {} ", JsonUtils.toJson(projectSteps)); - } - - - OmsProjectProgressResponse progress = omsProjectOpenApiService.describeProjectProgress(projectRequest); - - return new ProjectStepResultChecker(progress, projectSteps, - onlineSchemaChangeProperties.isEnableFullVerify(), - onlineSchemaChangeProperties.getOms().getCheckProjectStepFailedTimeoutSeconds(), - checkFailedTimes) - .withCheckerVerifyResult(() -> listProjectFullVerifyResult(projectID, - databaseName, uid, omsProjectOpenApiService)) - .withResumeProject(() -> { - omsProjectOpenApiService.resumeProject(projectRequest); - return null; - }) - .getCheckerResult(); - } - - private static OmsProjectFullVerifyResultResponse listProjectFullVerifyResult(String projectId, String databaseName, - String uid, OmsProjectOpenApiService omsProjectOpenApiService) { - ListOmsProjectFullVerifyResultRequest request = new ListOmsProjectFullVerifyResultRequest(); - request.setProjectId(projectId); - request.setSourceSchemas(new String[] {databaseName}); - request.setDestSchemas(new String[] {databaseName}); - request.setStatus(new String[] {"FINISHED", "SUSPEND", "RUNNING"}); - request.setPageSize(10); - request.setPageNumber(1); - request.setUid(uid); - - OmsProjectFullVerifyResultResponse response = omsProjectOpenApiService.listProjectFullVerifyResult(request); - if (log.isDebugEnabled()) { - log.debug("Get project full verify result from projectOpenApiService is {} ", JsonUtils.toJson(response)); - } - return response; - } - - /** - * sleep with exception swallowed - * - * @param millionSeconds ms to sleep - */ - public static void sleep(long millionSeconds) { - try { - Thread.sleep(millionSeconds); - } catch (InterruptedException e) { // swallow exception - } - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/client/BaseOmsClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/client/BaseOmsClient.java index 0cc67d2f7c..219f357d93 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/client/BaseOmsClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/client/BaseOmsClient.java @@ -29,6 +29,7 @@ import org.springframework.web.client.RestTemplate; import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.annotations.VisibleForTesting; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.validate.ValidatorUtils; @@ -88,8 +89,10 @@ private ResponseEntity exchange(BaseOmsRequest request, String realUrl) return omsRestTemplate.exchange(realUrl, HttpMethod.POST, httpRequest, String.class); } - private OmsApiReturnResult resolveResponseEntity(ClientRequestParams requestParams, + @VisibleForTesting + protected OmsApiReturnResult resolveResponseEntity(ClientRequestParams requestParams, ResponseEntity responseEntity, TypeReference> typeReference) { + log.info("process oms request [{}] with response [{}]", requestParams, responseEntity.getBody()); if (responseEntity.getStatusCode() != HttpStatus.OK) { if (responseEntity.getStatusCode() == HttpStatus.REQUEST_TIMEOUT) { throw new OmsException(ErrorCodes.Timeout, responseEntity.toString(), null, @@ -100,7 +103,6 @@ private OmsApiReturnResult resolveResponseEntity(ClientRequestParams requ : ErrorCodes.BadRequest; throw new OmsException(errorCode, responseEntity.toString(), null, responseEntity.getStatusCode()); } - log.info("process oms request [{}] with response [{}]", requestParams, responseEntity.getBody()); OmsApiReturnResult result = JsonUtils.fromJsonIgnoreMissingProperty(responseEntity.getBody(), typeReference); if (result == null) { throw new UnexpectedException("Parse oms result occur error, result=" + responseEntity.getBody()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/ActionScheduler.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/ActionScheduler.java index 0e5985ca9f..dc3e2bb99d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/ActionScheduler.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/ActionScheduler.java @@ -70,7 +70,7 @@ public void submitFMSScheduler(ScheduleEntity scheduleEntity, Long scheduleTaskI JsonUtils.toJson(scheduleEntity.getJobParametersJson())); } catch (SchedulerException e) { throw new IllegalArgumentException(MessageFormat.format( - "Create a quartz job check oms project occur error, jobParameters ={0}", + "Create a quartz job check oms project occur error, jobParameters={0}", JsonUtils.toJson(scheduleEntity.getJobParametersJson())), e); } } @@ -79,9 +79,9 @@ public void cancelScheduler(Long scheduleId) { JobKey jobKey = QuartzKeyGenerator.generateJobKey(scheduleId, JobType.ONLINE_SCHEMA_CHANGE_COMPLETE); try { quartzJobService.deleteJob(jobKey); - log.info("Successfully delete job with jobKey {}", jobKey); + log.info("Successfully delete job with jobKey={}", jobKey); } catch (SchedulerException e) { - log.warn("Delete job occur error with jobKey =" + jobKey, e); + log.warn("Delete job occur error with jobKey={}", jobKey, e); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java index 751bc35196..63a5c8dde3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java @@ -18,7 +18,6 @@ import java.time.Duration; import java.time.Instant; import java.util.Collections; -import java.util.List; import java.util.Map; import javax.annotation.PostConstruct; @@ -26,14 +25,11 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import com.google.common.collect.Lists; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.validate.ValidatorUtils; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionUtil; -import com.oceanbase.odc.core.shared.constant.ErrorCode; -import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.FlowStatus; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.metadb.schedule.ScheduleEntity; @@ -41,7 +37,9 @@ import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.flow.BeanInjectedClassDelegate; import com.oceanbase.odc.service.flow.FlowInstanceService; +import com.oceanbase.odc.service.onlineschemachange.OnlineSchemaChangeFlowableTask; import com.oceanbase.odc.service.onlineschemachange.configuration.OnlineSchemaChangeProperties; import com.oceanbase.odc.service.onlineschemachange.fsm.ActionFsm; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; @@ -93,14 +91,6 @@ public abstract class OscActionFsmBase extends ActionFsm terminalErrorCodes = - Lists.newArrayList(ErrorCodes.BadArgument, ErrorCodes.BadRequest, - ErrorCodes.OmsConnectivityTestFailed, ErrorCodes.Unexpected, - ErrorCodes.OmsPreCheckFailed, ErrorCodes.OmsDataCheckInconsistent, - ErrorCodes.OmsParamError, ErrorCodes.OmsBindTargetNotFound, - ErrorCodes.OmsProjectExecutingFailed); - // register state change action @PostConstruct public abstract void init(); @@ -122,33 +112,91 @@ public String resolveState(OscActionContext context) { public void schedule(Long schedulerID, Long schedulerTaskID) { OscActionContext oscActionContext = getOSCContext(schedulerID, schedulerTaskID); String state = resolveState(oscActionContext); - // CLEAN_RESOURCE should always schedule - if (!StringUtils.equals(OscStates.CLEAN_RESOURCE.getState(), state) && - (isTaskExpired(oscActionContext) || isFlowInstanceFailed(state, oscActionContext))) { - // transfer from current state to clean resources - transferTaskStatesWithStates(state, OscStates.CLEAN_RESOURCE.getState(), null, - oscActionContext.getScheduleTask(), TaskStatus.CANCELED); + // try process task may in expired or canceled or abnormal state + // state should not in CLEAN_RESOURCE state + if (!StringUtils.equals(OscStates.CLEAN_RESOURCE.getState(), state) + && tryHandleInvalidTask(state, oscActionContext)) { return; } // do schedule schedule(oscActionContext); + // if schedule failed, transfer to abnormal state + syncFlowInstanceState(oscActionContext); } - public boolean isFlowInstanceFailed(String state, OscActionContext oscActionContext) { - // only check in monitor data task state - if (!StringUtils.equals(state, OscStates.MONITOR_DATA_TASK.getState())) { - return false; + /** + * process task with expired or canceled or abnormal + * + * @param state current state + * @param oscActionContext osc context + * @return true if task in expired or canceled state + */ + protected boolean tryHandleInvalidTask(String state, OscActionContext oscActionContext) { + // task has expired + // translate state to clean resource + // NOTICE: expired should been checked first + if (isTaskExpired(oscActionContext)) { + transferTaskStatesWithStates(state, OscStates.CLEAN_RESOURCE.getState(), null, + oscActionContext.getScheduleTask(), TaskStatus.FAILED); + return true; } + // task has been canceled + // translate state to clean resource + if (isTaskCanceled(oscActionContext)) { + transferTaskStatesWithStates(state, OscStates.CLEAN_RESOURCE.getState(), null, + oscActionContext.getScheduleTask(), TaskStatus.CANCELED); + return true; + } + // handle flow status with failed or canceled + FlowStatus flowStatus = getFlowInstanceStatus(oscActionContext); + if (isFlowStatusInvalid(flowStatus)) { + transferTaskStatesWithStates(state, OscStates.CLEAN_RESOURCE.getState(), null, + oscActionContext.getScheduleTask(), TaskStatus.CANCELED); + log.info("OSC: flow task has failed, transfer state to clean resources, flow task id={}, status={}", + oscActionContext.getParameter().getFlowInstanceId(), flowStatus); + return true; + } + // abnormal task do nothing + return isTaskAbnormal(oscActionContext); + } + + /** + * sync task status with flow instance + */ + public void syncFlowInstanceState(OscActionContext context) { + try { + OnlineSchemaChangeFlowableTask schemaChangeFlowableTask = + BeanInjectedClassDelegate + .instantiateDelegateWithoutPostConstructInvoke(OnlineSchemaChangeFlowableTask.class); + OnlineSchemaChangeParameters parameters = context.getParameter(); + schemaChangeFlowableTask.tryCompleteTask( + parameters.getFlowInstanceId(), parameters.getFlowTaskID(), + context.getSchedule().getId()); + } catch (Throwable e) { + log.warn("meet unhandled exception: cancel scheduler", e); + actionScheduler.cancelScheduler(context.getSchedule().getId()); + throw new RuntimeException(e); + } + } + + /** + * flow state in failed or canceled or abnormal + * + * @param oscActionContext + * @return + */ + public FlowStatus getFlowInstanceStatus(OscActionContext oscActionContext) { Long flowInstanceID = oscActionContext.getParameter().getFlowInstanceId(); Map statusMap = flowInstanceService.getStatus(Collections.singleton(flowInstanceID)); - FlowStatus taskStatus = statusMap.get(flowInstanceID); - // not found or failed - boolean ret = (null == taskStatus || taskStatus == FlowStatus.EXECUTION_FAILED); - if (ret) { - log.info("OSC: flow task has failed, transfer state to clean resources, flow task id {}, status {}", - flowInstanceID, taskStatus); + return statusMap.get(flowInstanceID); + } + + // not found or failed + private boolean isFlowStatusInvalid(FlowStatus flowStatus) { + if (null == flowStatus) { + return true; } - return ret; + return flowStatus == FlowStatus.EXECUTION_FAILED || flowStatus == FlowStatus.CANCELLED; } /** @@ -174,18 +222,6 @@ public void start(Long schedulerID, Long schedulerTaskID) { actionScheduler.submitFMSScheduler(scheduleEntity, schedulerTaskID); } - public void cancel(Long schedulerID, Long schedulerTaskID) { - ScheduleEntity scheduleEntity = scheduleService.nullSafeGetById(schedulerID); - ScheduleTaskEntity scheduleTaskEntity = scheduleTaskService.nullSafeGetById(schedulerTaskID); - OnlineSchemaChangeScheduleTaskParameters parameters = JsonUtils.fromJson(scheduleTaskEntity.getParametersJson(), - OnlineSchemaChangeScheduleTaskParameters.class); - String currentState = parameters.getState(); - transferTaskStatesWithStates(null, OscStates.CLEAN_RESOURCE.getState(), - "CANCEL", scheduleTaskEntity, TaskStatus.CANCELED); - // try submit scheduler in case task may in failed state - actionScheduler.submitFMSScheduler(scheduleEntity, schedulerTaskID); - } - /** * check if task should continue * @@ -194,32 +230,37 @@ public void cancel(Long schedulerID, Long schedulerTaskID) { */ protected boolean isTaskExpired(OscActionContext context) { ScheduleTaskEntity scheduleTask = context.getScheduleTask(); - OnlineSchemaChangeScheduleTaskParameters parameters = - parseOnlineSchemaChangeScheduleTaskParameters(scheduleTask.getParametersJson()); // check task is expired Long scheduleId = context.getSchedule().getId(); Long scheduleTaskId = scheduleTask.getId(); Duration between = Duration.between(scheduleTask.getCreateTime().toInstant(), Instant.now()); - log.info("Schedule id {} to check schedule task status with schedule task id {}", scheduleId, scheduleTaskId); + log.info("Schedule id={} to check schedule task status with schedule task id={}", scheduleId, scheduleTaskId); if (between.toMillis() / 1000 > oscTaskExpiredAfterSeconds) { // schedule to clean resource // has canceled - log.info("Schedule task id {} is expired after {} seconds, so cancel the scheduleTaskId ", + log.info("Schedule task id={} is expired after {} seconds, so cancel the scheduleTaskId ", scheduleTaskId, oscTaskExpiredAfterSeconds); return true; + } else { + return false; } - // task has been canceled, clean resources must has been done - if (scheduleTask.getStatus() == TaskStatus.CANCELED) { - return true; - } - return false; + } + + public boolean isTaskAbnormal(OscActionContext context) { + ScheduleTaskEntity scheduleTask = context.getScheduleTask(); + return TaskStatus.ABNORMAL == scheduleTask.getStatus(); + } + + public boolean isTaskCanceled(OscActionContext context) { + ScheduleTaskEntity scheduleTask = context.getScheduleTask(); + return TaskStatus.CANCELED == scheduleTask.getStatus(); } @Override public void onActionComplete(String currentState, String nextState, String extraInfo, OscActionContext context) { if (StringUtils.equals(nextState, OscStates.COMPLETE.getState())) { - log.info("OCS: complete state reached, delete scheduler, prev state {}, schedule id {}", currentState, + log.info("OCS: complete state reached, delete scheduler, prev state={}, schedule id={}", currentState, context.getSchedule().getId()); actionScheduler.cancelScheduler(context.getSchedule().getId()); } @@ -269,20 +310,18 @@ public void transferTaskStatesWithStates(String currentState, String nextState, } scheduleTaskEntity.setParametersJson(JsonUtils.toJson(parameters)); scheduleTaskEntity.setStatus(taskStatus); - if (TaskStatus.DONE == taskStatus || TaskStatus.FAILED == taskStatus || TaskStatus.CANCELED == taskStatus) { + if (TaskStatus.DONE == taskStatus || TaskStatus.CANCELED == taskStatus) { scheduleTaskEntity.setProgressPercentage(100.0); } - log.info("Successfully update schedule task id {} set status {}", scheduleTaskEntity.getId(), taskStatus); + log.info("Successfully update schedule task id={} set status={}", scheduleTaskEntity.getId(), taskStatus); scheduleTaskRepository.update(scheduleTaskEntity); - } @Override public void handleException(OscActionContext context, Throwable e) { // we hope in resume mode continue, not drop oms project // not do state transfer - actionScheduler.cancelScheduler(context.getSchedule().getId()); - scheduleTaskRepository.updateStatusById(context.getScheduleTask().getId(), TaskStatus.FAILED); + scheduleTaskRepository.updateStatusById(context.getScheduleTask().getId(), TaskStatus.ABNORMAL); } protected OscActionContext getOSCContext(Long scheduleId, Long scheduleTaskId) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsCleanResourcesAction.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsCleanResourcesAction.java index ada7d4002f..7ee98a397b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsCleanResourcesAction.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsCleanResourcesAction.java @@ -68,7 +68,7 @@ public OscActionResult execute(OscActionContext context) throws Exception { boolean released = checkAndReleaseProject(taskParameters.getOmsProjectId(), taskParameters.getUid()); if (!released) { // try release again - log.info("OCS: release OMS project failed, try it again. OMS project ID {}, uid {}", + log.info("OCS: release OMS project failed, try it again. OMS project ID={}, uid={}", taskParameters.getOmsProjectId(), taskParameters.getUid()); return new OscActionResult(OscStates.CLEAN_RESOURCE.getState(), null, OscStates.CLEAN_RESOURCE.getState()); } @@ -84,7 +84,7 @@ protected OscActionResult determinateNextState(ScheduleTaskEntity scheduleTask, } // if task state is in cancel state, stop and transfer to complete state if (scheduleTask.getStatus() == TaskStatus.CANCELED) { - log.info("Because task is canceled, so delete quartz job {}", scheduleId); + log.info("Because task is canceled, so delete quartz job={}", scheduleId); // cancel as complete return new OscActionResult(OscStates.CLEAN_RESOURCE.getState(), null, OscStates.COMPLETE.getState()); } @@ -96,7 +96,7 @@ protected OscActionResult determinateNextState(ScheduleTaskEntity scheduleTask, // try schedule next task return new OscActionResult(OscStates.CLEAN_RESOURCE.getState(), null, OscStates.YIELD_CONTEXT.getState()); } else { - log.info("Because error strategy is abort, so delete quartz job {}", scheduleId); + log.info("Because error strategy is abort, so delete quartz job={}", scheduleId); // not continue for remain state, transfer to complete state return new OscActionResult(OscStates.CLEAN_RESOURCE.getState(), null, OscStates.COMPLETE.getState()); } @@ -110,7 +110,7 @@ protected boolean checkAndReleaseProject(String omsProjectId, String uid) { OmsProjectControlRequest controlRequest = new OmsProjectControlRequest(); controlRequest.setId(omsProjectId); controlRequest.setUid(uid); - log.info("Oms project {} has not released, try to release it.", omsProjectId); + log.info("Oms project={} has not released, try to release it.", omsProjectId); return checkAndReleaseProject(controlRequest); } @@ -135,7 +135,7 @@ protected boolean checkAndReleaseProject(OmsProjectControlRequest projectControl } if (!released) { - log.info("Oms project {} has not released, try to release it.", projectControl.getId()); + log.info("Oms project={} has not released, try to release it.", projectControl.getId()); doReleaseOmsResource(controlRequest); } return released; @@ -153,9 +153,9 @@ private void doReleaseOmsResource(OmsProjectControlRequest projectControlRequest projectOpenApiService.stopProject(projectControlRequest); } projectOpenApiService.releaseProject(projectControlRequest); - log.info("Release oms project, id {}", projectControlRequest.getId()); + log.info("Release oms project, id={}", projectControlRequest.getId()); } catch (Throwable ex) { - log.warn("Failed to release oms project, id {}, occur error {}", projectControlRequest.getId(), + log.warn("Failed to release oms project, id={}, occur error={}", projectControlRequest.getId(), ex.getMessage()); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsModifyDataTaskAction.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsModifyDataTaskAction.java index 24286cef5d..599be4ef5a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsModifyDataTaskAction.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsModifyDataTaskAction.java @@ -58,7 +58,7 @@ public OmsModifyDataTaskAction(@NotNull OmsProjectOpenApiService projectOpenApiS public OscActionResult execute(OscActionContext context) throws Exception { ScheduleTaskEntity scheduleTask = context.getScheduleTask(); - log.debug("Start execute {}, schedule task id {}", getClass().getSimpleName(), scheduleTask.getId()); + log.debug("Start execute {}, schedule task id={}", getClass().getSimpleName(), scheduleTask.getId()); OnlineSchemaChangeScheduleTaskParameters taskParameter = context.getTaskParameter(); OnlineSchemaChangeParameters inputParameters = context.getParameter(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsMonitorDataTaskAction.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsMonitorDataTaskAction.java index 4a334e3abf..1053b5ca2a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsMonitorDataTaskAction.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsMonitorDataTaskAction.java @@ -67,7 +67,7 @@ public OmsMonitorDataTaskAction(@NotNull OmsProjectOpenApiService projectOpenApi @Override public OscActionResult execute(OscActionContext context) throws Exception { ScheduleTaskEntity scheduleTask = context.getScheduleTask(); - log.debug("Start execute {}, schedule task id {}", getClass().getSimpleName(), scheduleTask.getId()); + log.debug("Start execute {}, schedule task id={}", getClass().getSimpleName(), scheduleTask.getId()); OnlineSchemaChangeScheduleTaskParameters taskParameter = context.getTaskParameter(); OnlineSchemaChangeParameters inputParameters = context.getParameter(); @@ -137,7 +137,7 @@ protected OscActionResult handleOmsProjectStepResult(OscActionContext context, P result.setManualSwapTableEnabled(true); scheduleTask.setResultJson(JsonUtils.toJson(result)); } - log.info("OSC: oms project ready, wait manual swap table triggered, task id {}", + log.info("OSC: oms project ready, wait manual swap table triggered, task id={}", scheduleTask.getId()); // manual swap table not set, keep waiting return new OscActionResult(OscStates.MONITOR_DATA_TASK.getState(), null, diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java index e0ade26ed7..bc1c3347e8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java @@ -77,7 +77,7 @@ public OscActionResult execute(OscActionContext context) throws Exception { } // begin swap table ScheduleTaskEntity scheduleTask = context.getScheduleTask(); - log.info("Start execute {}, schedule task id {}", getClass().getSimpleName(), scheduleTask.getId()); + log.info("Start execute {}, schedule task id={}", getClass().getSimpleName(), scheduleTask.getId()); OnlineSchemaChangeScheduleTaskParameters taskParameters = context.getTaskParameter(); PreConditions.notNull(taskParameters, "OnlineSchemaChangeScheduleTaskParameters is null"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleMapper.java index 5dfb1c32d2..e722633375 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleMapper.java @@ -26,6 +26,7 @@ import com.oceanbase.odc.service.dlm.model.DataArchiveParameters; import com.oceanbase.odc.service.dlm.model.DataDeleteParameters; import com.oceanbase.odc.service.loaddata.model.LoadDataParameters; +import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; import com.oceanbase.odc.service.sqlplan.model.SqlPlanParameters; /** @@ -58,6 +59,8 @@ default ScheduleTaskParameters entityToParameters(ScheduleEntity entity) { return JsonUtils.fromJson(entity.getJobParametersJson(), LogicalDatabaseChangeParameters.class); case LOAD_DATA: return JsonUtils.fromJson(entity.getJobParametersJson(), LoadDataParameters.class); + case ONLINE_SCHEMA_CHANGE_COMPLETE: + return JsonUtils.fromJson(entity.getJobParametersJson(), OnlineSchemaChangeParameters.class); default: throw new UnsupportedException(); } diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java new file mode 100644 index 0000000000..ee8e25290a --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.onlineschemachange; + +import java.util.Arrays; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; + +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; +import com.oceanbase.odc.metadb.task.TaskEntity; +import com.oceanbase.odc.service.flow.task.model.OnlineSchemaChangeTaskResult; +import com.oceanbase.odc.service.task.TaskService; + +/** + * @author longpeng.zlp + * @date 2024/8/23 10:39 + */ +public class OnlineSchemaChangeFlowableTaskTest { + private static String resultJson = "{" + + "\"fullLogDownloadUrl\":null," + + "\"tasks\":[" + + " {" + + " \"createTime\":1724291469000," + + " \"executor\":null,\"fireTime\":1724291470000," + + " \"id\":65,\"jobGroup\":\"ONLINE_SCHEMA_CHANGE_COMPLETE\"," + + " \"jobId\":null,\"jobName\":\"64\"," + + " \"parameters\":" + + " {" + + " \"continueOnError\":false," + + " \"delimiter\":\";\"," + + " \"errorStrategy\":null," + + " \"flowInstanceId\":null," + + " \"flowTaskID\":null," + + " \"lockTableTimeOutSeconds\":null," + + " \"lockUsers\":null," + + " \"originTableCleanStrategy\":null," + + " \"rateLimitConfig\":{\"dataSizeLimit\":null,\"rowLimit\":null}," + + " \"sqlContent\":null," + + " \"sqlType\":null," + + " \"swapTableNameRetryTimes\":null," + + " \"swapTableType\":null}," + + " \"progressPercentage\":90.00," + + " \"resultJson\":\"{\\\"checkFailedTime\\\":{},\\\"currentStep\\\":\\\"TRANSFER_APP_SWITCH\\\",\\\"currentStepStatus\\\":\\\"RUNNING\\\",\\\"dialectType\\\":\\\"OB_MYSQL\\\",\\\"fullTransferEstimatedCount\\\":13,\\\"fullTransferFinishedCount\\\":13,\\\"fullTransferProgressPercentage\\\":100.00,\\\"fullVerificationProgressPercentage\\\":0.00,\\\"fullVerificationResult\\\":\\\"UNCHECK\\\",\\\"fullVerificationResultDescription\\\":\\\"未校验\\\",\\\"manualSwapTableEnabled\\\":true,\\\"manualSwapTableStarted\\\":false,\\\"newTableDdl\\\":null,\\\"oldTableName\\\":\\\"_ghost1_osc_old_\\\",\\\"originTableDdl\\\":\\\"CREATE TABLE `ghost1` (\\\\n `id` int(11) NOT NULL AUTO_INCREMENT,\\\\n `age` int(11) DEFAULT NULL,\\\\n PRIMARY KEY (`id`)\\\\n) AUTO_INCREMENT = 15000001 DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'zstd_1.3.8' REPLICA_NUM = 1 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 0\\\",\\\"originTableName\\\":\\\"ghost1\\\",\\\"precheckResult\\\":\\\"FINISHED\\\",\\\"precheckResultDescription\\\":null" + + " }\"," + + " \"status\":\"RUNNING\"," + + " \"updateTime\":1724291750000" + + " }" + + " ]" + + "}"; + + // test deserialize + @Test + public void testOlineSchemaChangeTaskResultDeserialize() { + String json = resultJson; + OnlineSchemaChangeTaskResult onlineSchemaChangeTaskResult = + JsonUtils.fromJson(json, OnlineSchemaChangeTaskResult.class); + Assert.assertNotNull(onlineSchemaChangeTaskResult); + } + + // test flowable task update progress + @Test + public void testOnlineSchemaChangeFlowableTaskUpdateProgress() { + MockOSCFlowTask mockOSCFlowTask = new MockOSCFlowTask(); + TaskService taskService = Mockito.mock(TaskService.class); + // prepare mock + TaskEntity taskEntity = new TaskEntity(); + taskEntity.setStatus(TaskStatus.RUNNING); + taskEntity.setResultJson(resultJson); + Mockito.when(taskService.detail(1L)).thenReturn(taskEntity); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskEntity.class); + // do method call expect task update + mockOSCFlowTask.onProgressUpdate(1L, taskService); + Mockito.verify(taskService).update(argumentCaptor.capture()); + TaskEntity task = argumentCaptor.getValue(); + OnlineSchemaChangeTaskResult onlineSchemaChangeTaskResult = + JsonUtils.fromJson(task.getResultJson(), OnlineSchemaChangeTaskResult.class); + // check update result json + Assert.assertEquals(onlineSchemaChangeTaskResult.getTasks().size(), 1); + Assert.assertEquals(onlineSchemaChangeTaskResult.getTasks().get(0).getId().longValue(), 1024); + } + + private static final class MockOSCFlowTask extends OnlineSchemaChangeFlowableTask { + private Page createSchedulerTaskEntity() { + Page mockPage = Mockito.mock(Page.class); + ScheduleTaskEntity scheduleTask = new ScheduleTaskEntity(); + scheduleTask.setId(1L); + scheduleTask.setStatus(TaskStatus.RUNNING); + ScheduleTaskEntity scheduleTask2 = new ScheduleTaskEntity(); + scheduleTask2.setStatus(TaskStatus.PREPARING); + scheduleTask2.setId(1024L); + List scheduleTaskEntities = Arrays.asList(scheduleTask); + Mockito.when(mockPage.getSize()).thenReturn(scheduleTaskEntities.size()); + Mockito.when(mockPage.iterator()).thenReturn(scheduleTaskEntities.iterator()); + Mockito.when(mockPage.getContent()).thenReturn(Arrays.asList(scheduleTask2)); + return mockPage; + } + + protected Page list(Pageable pageable, Long scheduleId) { + return createSchedulerTaskEntity(); + } + } +} diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oms/client/OmsClientTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oms/client/OmsClientTest.java new file mode 100644 index 0000000000..503d71db58 --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oms/client/OmsClientTest.java @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.onlineschemachange.oms.client; + +import java.util.List; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.RestTemplate; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.oceanbase.odc.service.onlineschemachange.configuration.OnlineSchemaChangeProperties; +import com.oceanbase.odc.service.onlineschemachange.configuration.OnlineSchemaChangeProperties.OmsProperties; +import com.oceanbase.odc.service.onlineschemachange.exception.OmsException; +import com.oceanbase.odc.service.onlineschemachange.oms.request.ApiReturnResult; +import com.oceanbase.odc.service.onlineschemachange.oms.request.OmsApiReturnResult; +import com.oceanbase.odc.service.onlineschemachange.oms.response.OmsProjectResponse; + +/** + * @author longpeng.zlp + * @date 2024/9/11 14:49 + */ +public class OmsClientTest { + private final String badResponseBody = "{\n" + + " \"code\": \"GHANA-OPERIE000000\",\n" + + " \"errorDetail\": {\n" + + " \"code\": \"GHANA-OPERIE000000\",\n" + + " \"message\": \"传输实例正在创建中，通常需要 5~10 分钟，请稍后重试。\",\n" + + " \"messageMcmsContext\": {\n" + + " \"message\": \"传输实例正在创建中，通常需要 5~10 分钟，请稍后重试。\"\n" + + " },\n" + + " \"messageMcmsKey\": \"GHANA-OPERIE000000\",\n" + + " \"requestId\": \"xxxx\"\n" + + " },\n" + + " \"message\": \"[GHANA-OPERIE000000]: 服务内部错误。\",\n" + + " \"requestId\": \"xxxx\",\n" + + " \"success\": false\n" + + "}"; + private final String listProjectResult = "{\n" + + " \"success\": true,\n" + + " \"errorDetail\": null,\n" + + " \"code\": null,\n" + + " \"message\": null,\n" + + " \"advice\": null,\n" + + " \"requestId\": \"****\",\n" + + " \"pageNumber\": 1,\n" + + " \"pageSize\": 10,\n" + + " \"totalCount\": 1,\n" + + " \"cost\": \"13 ms\",\n" + + " \"data\": [\n" + + " {\n" + + " \"workerGradeId\": \"myworkerID\",\n" + + " \"id\": \"myID\",\n" + + " \"type\": \"MIGRATION\",\n" + + " \"name\": \"****\",\n" + + " \"labels\": null,\n" + + " \"owner\": \"xxxx\",\n" + + " \"importance\": \"LOW\",\n" + + " \"status\": \"RUNNING\",\n" + + " \"gmtCreate\": \"2024-09-13T09:08:34\",\n" + + " \"gmtModified\": \"2024-09-18T07:01:52\",\n" + + " \"gmtStart\": \"2024-09-13T09:08:35\",\n" + + " \"gmtFinish\": null,\n" + + " \"destConnId\": null,\n" + + " \"isMerging\": false,\n" + + " \"isModifying\": false,\n" + + " \"isSubProject\": false,\n" + + " \"sourceEndpointType\": \"MYSQL\",\n" + + " \"sinkEndpointType\": \"OB_MYSQL\",\n" + + " \"transferMapping\": null,\n" + + " \"commonTransferConfig\": null,\n" + + " \"enableStructTransfer\": false,\n" + + " \"structTransferConfig\": null,\n" + + " \"enableFullTransfer\": false,\n" + + " \"enableFullVerify\": false,\n" + + " \"fullTransferConfig\": null,\n" + + " \"enableIncrTransfer\": true,\n" + + " \"enableIncrVerify\": false,\n" + + " \"enableReverseIncrTransfer\": false,\n" + + " \"incrTransferConfig\": null,\n" + + " \"sourceConnectInfo\": {\n" + + " \"id\": \"*****\",\n" + + " \"endpointName\": \"****\",\n" + + " \"endpointId\": \"*****\",\n" + + " \"endpointSide\": null,\n" + + " \"dbEngine\": \"MYSQL_PUBLIC\",\n" + + " \"connectionInfo\": null,\n" + + " \"username\": \"root\",\n" + + " \"version\": \"5.7.27-log\",\n" + + " \"timezone\": \"UTC\",\n" + + " \"charset\": \"utf8mb4\",\n" + + " \"nlsLengthSemantics\": null,\n" + + " \"operatingSystem\": \"Linux\",\n" + + " \"region\": \"cn-shanghai\",\n" + + " \"ocpName\": \"\",\n" + + " \"connExtraAttributes\": null,\n" + + " \"owner\": \"admin\",\n" + + " \"resourceOwner\": \"admin\",\n" + + " \"host\": \"****\",\n" + + " \"port\": 3306\n" + + " },\n" + + " \"sinkConnectInfo\": {\n" + + " \"id\": \"****\",\n" + + " \"endpointName\": \"****\",\n" + + " \"endpointId\": \"*****\",\n" + + " \"endpointSide\": null,\n" + + " \"dbEngine\": \"OB_MYSQL_PUBLIC\",\n" + + " \"connectionInfo\": null,\n" + + " \"username\": \"*****\",\n" + + " \"version\": \"3.2.4.8\",\n" + + " \"timezone\": \"+08:00\",\n" + + " \"charset\": \"utf8mb4\",\n" + + " \"nlsLengthSemantics\": null,\n" + + " \"operatingSystem\": null,\n" + + " \"region\": \"*****\",\n" + + " \"ocpName\": \"\",\n" + + " \"connExtraAttributes\": {\n" + + " \"cluster\": \"*****\",\n" + + " \"tenant\": \"oms_mysql\",\n" + + " \"isLogicSource\": false,\n" + + " \"useLogProxy\": false,\n" + + " \"drcUser\": \"root\",\n" + + " \"configUrl\": \"******\",\n" + + " \"logProxyIp\": null,\n" + + " \"logProxyPort\": null,\n" + + " \"noUserAuth\": false\n" + + " },\n" + + " \"owner\": \"****\",\n" + + " \"resourceOwner\": \"****\",\n" + + " \"host\": \"****\",\n" + + " \"port\": 56569\n" + + " },\n" + + " \"steps\": [\n" + + " {\n" + + " \"order\": 1,\n" + + " \"name\": \"TRANSFER_PRECHECK\",\n" + + " \"description\": \"预检查\",\n" + + " \"status\": \"FINISHED\",\n" + + " \"extraInfo\": {\n" + + " \"errorDetails\": null,\n" + + " \"errorCode\": null,\n" + + " \"errorMsg\": null,\n" + + " \"errorParam\": null,\n" + + " \"failedTime\": null\n" + + " },\n" + + " \"startTime\": \"2024-09-13T09:08:38\",\n" + + " \"finishTime\": \"2024-09-13T09:08:38\",\n" + + " \"progress\": 100,\n" + + " \"stepInfo\": null\n" + + " },\n" + + " {\n" + + " \"order\": 3,\n" + + " \"name\": \"INCR_TRANSFER\",\n" + + " \"description\": \"增量同步\",\n" + + " \"status\": \"MONITORING\",\n" + + " \"extraInfo\": {\n" + + " \"errorDetails\": null,\n" + + " \"errorCode\": null,\n" + + " \"errorMsg\": null,\n" + + " \"errorParam\": null,\n" + + " \"failedTime\": null\n" + + " },\n" + + " \"startTime\": \"2024-09-13T09:09:17\",\n" + + " \"finishTime\": null,\n" + + " \"progress\": 100,\n" + + " \"stepInfo\": {\n" + + " \"incrTimestampCheckpoint\": 1726642901,\n" + + " \"checkpointSampleTimestamp\": 1726642905,\n" + + " \"enableIncrStatistics\": null\n" + + " }\n" + + " },\n" + + " {\n" + + " \"order\": 4,\n" + + " \"name\": \"TRANSFER_APP_SWITCH\",\n" + + " \"description\": \"正向切换\",\n" + + " \"status\": \"RUNNING\",\n" + + " \"extraInfo\": {\n" + + " \"errorDetails\": null,\n" + + " \"errorCode\": null,\n" + + " \"errorMsg\": null,\n" + + " \"errorParam\": null,\n" + + " \"failedTime\": null\n" + + " },\n" + + " \"startTime\": \"2024-09-13T09:09:23\",\n" + + " \"finishTime\": null,\n" + + " \"progress\": 0,\n" + + " \"stepInfo\": {\n" + + " \"checkpointSampleTimestamp\": null\n" + + " }\n" + + " }\n" + + " ],\n" + + " \"extraInfo\": null,\n" + + " \"alarmStats\": {\n" + + " \"target\": null,\n" + + " \"alarming\": false,\n" + + " \"recentlyTriggerCount\": null,\n" + + " \"ruleToRecentlyTriggerCount\": null,\n" + + " \"alarmContent\": null,\n" + + " \"openMonitor\": null\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"; + private DefaultPrivateCloudOmsClient client; + private ClientRequestParams requestParams; + private ResponseEntity responseEntity; + + @Before + public void init() { + OnlineSchemaChangeProperties properties = new OnlineSchemaChangeProperties(); + OmsProperties omsProperties = new OmsProperties(); + omsProperties.setUrl("127.0.0.1"); + properties.setOms(omsProperties); + client = new DefaultPrivateCloudOmsClient( + properties, + Mockito.mock(RestTemplate.class)); + requestParams = new ClientRequestParams(); + requestParams.setAction("CreateDataSource"); + responseEntity = (ResponseEntity) Mockito.mock(ResponseEntity.class); + } + + @Test(expected = OmsException.class) + public void testResolveErrorDetailFromOmsResponse() { + Mockito.when(responseEntity.getStatusCode()).thenReturn(HttpStatus.OK); + Mockito.when(responseEntity.getBody()).thenReturn(badResponseBody); + ApiReturnResult createDataSourceBadResponse = client.resolveResponseEntity(requestParams, + responseEntity, new TypeReference>() {}); + } + + @Test + public void testListOmsProject() { + Mockito.when(responseEntity.getStatusCode()).thenReturn(HttpStatus.OK); + Mockito.when(responseEntity.getBody()).thenReturn(listProjectResult); + ApiReturnResult> listObjectResponse = client.resolveResponseEntity(requestParams, + responseEntity, new TypeReference>>() {}); + Assert.assertEquals(listObjectResponse.getData().size(), 1); + OmsProjectResponse omsProjectResponse = listObjectResponse.getData().get(0); + Assert.assertEquals(omsProjectResponse.getId(), "myID"); + Assert.assertEquals(omsProjectResponse.getWorkerGradeId(), "myworkerID"); + } +} diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBaseTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBaseTest.java new file mode 100644 index 0000000000..9493f7b28d --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBaseTest.java @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.onlineschemachange.oscfms; + +import java.util.Collections; +import java.util.Date; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.core.shared.constant.FlowStatus; +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.metadb.schedule.ScheduleEntity; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; +import com.oceanbase.odc.service.flow.FlowInstanceService; +import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; +import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeScheduleTaskParameters; +import com.oceanbase.odc.service.onlineschemachange.oscfms.state.OscStates; + +/** + * @author longpeng.zlp + * @date 2024/9/23 11:56 + */ +public class OscActionFsmBaseTest { + private long expireTimeInSeconds = 3600; + private OscActionFsm oscActionFsm = new OscActionFsm(); + private OscActionContext context = new OscActionContext(); + private ScheduleTaskEntity scheduleTaskEntity = new ScheduleTaskEntity(); + private ScheduleEntity scheduleEntity = new ScheduleEntity(); + private String state = OscStates.MONITOR_DATA_TASK.getState(); + private OnlineSchemaChangeScheduleTaskParameters parameters = new OnlineSchemaChangeScheduleTaskParameters(); + private OnlineSchemaChangeParameters onlineSchemaChangeParameters = new OnlineSchemaChangeParameters(); + private ScheduleTaskRepository scheduleTaskRepository; + private FlowInstanceService flowInstanceService; + + @Before + public void init() { + context.setSchedule(scheduleEntity); + context.setScheduleTask(scheduleTaskEntity); + context.setParameter(onlineSchemaChangeParameters); + scheduleEntity.setId(1L); + scheduleTaskEntity.setId(1024L); + scheduleTaskEntity.setStatus(TaskStatus.RUNNING); + parameters.setState(OscStates.MONITOR_DATA_TASK.getState()); + onlineSchemaChangeParameters.setFlowInstanceId(1L); + scheduleTaskEntity.setParametersJson(JsonUtils.toJson(parameters)); + scheduleTaskRepository = Mockito.mock(ScheduleTaskRepository.class); + flowInstanceService = Mockito.mock(FlowInstanceService.class); + oscActionFsm.scheduleTaskRepository = scheduleTaskRepository; + oscActionFsm.flowInstanceService = flowInstanceService; + oscActionFsm.oscTaskExpiredAfterSeconds = expireTimeInSeconds; + } + + @Test + public void testTaskExpired() { + // not expired + scheduleTaskEntity.setCreateTime(new Date(System.currentTimeMillis())); + Assert.assertFalse(oscActionFsm.isTaskExpired(context)); + // expired + scheduleTaskEntity.setCreateTime(new Date(System.currentTimeMillis() - expireTimeInSeconds * 1000 * 10)); + Assert.assertTrue(oscActionFsm.isTaskExpired(context)); + } + + @Test + public void testTryHandleInvalidTaskWithTaskExpired() { + // expired + scheduleTaskEntity.setCreateTime(new Date(System.currentTimeMillis() - expireTimeInSeconds * 1000 * 10)); + boolean handleResult = oscActionFsm.tryHandleInvalidTask(state, context); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(ScheduleTaskEntity.class); + Assert.assertTrue(handleResult); + Mockito.verify(scheduleTaskRepository).update(argumentCaptor.capture()); + ScheduleTaskEntity toVerify = argumentCaptor.getValue(); + Assert.assertEquals(toVerify.getStatus(), TaskStatus.FAILED); + checkState(toVerify, OscStates.CLEAN_RESOURCE.getState()); + } + + @Test + public void testTryHandleInvalidTaskWithTaskCanceled() { + // not expired + scheduleTaskEntity.setCreateTime(new Date(System.currentTimeMillis())); + scheduleTaskEntity.setStatus(TaskStatus.CANCELED); + boolean handleResult = oscActionFsm.tryHandleInvalidTask(state, context); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(ScheduleTaskEntity.class); + Assert.assertTrue(handleResult); + Mockito.verify(scheduleTaskRepository).update(argumentCaptor.capture()); + ScheduleTaskEntity toVerify = argumentCaptor.getValue(); + Assert.assertEquals(toVerify.getStatus(), TaskStatus.CANCELED); + checkState(toVerify, OscStates.CLEAN_RESOURCE.getState()); + } + + @Test + public void testTryHandleInvalidTaskWithFlowCanceled() { + // not expired + scheduleTaskEntity.setCreateTime(new Date(System.currentTimeMillis())); + scheduleTaskEntity.setStatus(TaskStatus.RUNNING); + Mockito.when(flowInstanceService.getStatus(ArgumentMatchers.any())) + .thenReturn(Collections.singletonMap(1L, FlowStatus.CANCELLED)); + boolean handleResult = oscActionFsm.tryHandleInvalidTask(state, context); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(ScheduleTaskEntity.class); + Assert.assertTrue(handleResult); + Mockito.verify(scheduleTaskRepository).update(argumentCaptor.capture()); + ScheduleTaskEntity toVerify = argumentCaptor.getValue(); + Assert.assertEquals(toVerify.getStatus(), TaskStatus.CANCELED); + checkState(toVerify, OscStates.CLEAN_RESOURCE.getState()); + } + + @Test + public void testTryHandleInvalidTaskWithTaskAbnormal() { + // not expired + scheduleTaskEntity.setCreateTime(new Date(System.currentTimeMillis())); + scheduleTaskEntity.setStatus(TaskStatus.ABNORMAL); + Mockito.when(flowInstanceService.getStatus(ArgumentMatchers.any())) + .thenReturn(Collections.singletonMap(1L, FlowStatus.EXECUTION_ABNORMAL)); + boolean handleResult = oscActionFsm.tryHandleInvalidTask(state, context); + Assert.assertTrue(handleResult); + Mockito.verify(scheduleTaskRepository, Mockito.never()).update(ArgumentMatchers.any()); + } + + @Test + public void testTryHandleInvalidTaskNormal() { + // not expired + scheduleTaskEntity.setCreateTime(new Date(System.currentTimeMillis())); + scheduleTaskEntity.setStatus(TaskStatus.RUNNING); + Mockito.when(flowInstanceService.getStatus(ArgumentMatchers.any())) + .thenReturn(Collections.singletonMap(1L, FlowStatus.EXECUTING)); + boolean handleResult = oscActionFsm.tryHandleInvalidTask(state, context); + Assert.assertFalse(handleResult); + Mockito.verify(scheduleTaskRepository, Mockito.never()).update(ArgumentMatchers.any()); + } + + private void checkState(ScheduleTaskEntity scheduleTask, String expectState) { + OnlineSchemaChangeScheduleTaskParameters parameters = + JsonUtils.fromJson(scheduleTask.getParametersJson(), OnlineSchemaChangeScheduleTaskParameters.class); + Assert.assertEquals(expectState, parameters.getState()); + } +} From 513e60ba062531e0106fea66f2cde05979f1f2d2 Mon Sep 17 00:00:00 2001 From: LioRoger Date: Sun, 29 Sep 2024 16:30:00 +0800 Subject: [PATCH 006/118] feat(osc): check ghost table not exist before dispatch flow task (#3605) --- .../onlineschemachange/OscTableUtil.java | 49 +++++++++++++++++++ .../oscfms/action/CreateGhostTableAction.java | 39 ++------------- .../action/oms/OmsCleanResourcesAction.java | 34 +++++++++++-- .../oscfms/action/oms/OmsSwapTableAction.java | 2 +- .../OnlineSchemaChangeValidator.java | 18 ++++--- 5 files changed, 95 insertions(+), 47 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscTableUtil.java diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscTableUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscTableUtil.java new file mode 100644 index 0000000000..78a48c7c6e --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscTableUtil.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.onlineschemachange; + +import java.util.List; + +import org.apache.commons.collections4.CollectionUtils; + +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.service.db.browser.DBObjectOperators; +import com.oceanbase.odc.service.db.browser.DBSchemaAccessors; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +import lombok.extern.slf4j.Slf4j; + +/** + * util to operate osc table + * + * @author longpeng.zlp + * @date 2024/9/27 17:01 + */ +@Slf4j +public class OscTableUtil { + public static void dropNewTableIfExits(String databaseName, String tableName, ConnectionSession session) { + List list = DBSchemaAccessors.create(session) + .showTablesLike(databaseName, tableName); + // Drop new table if exists + if (CollectionUtils.isNotEmpty(list)) { + DBObjectOperators.create(session) + .drop(DBObjectType.TABLE, databaseName, tableName); + log.info("drop table {}.{}", databaseName, tableName); + } else { + log.info("table {}.{} not existed, ignore drop operation", databaseName, tableName); + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/CreateGhostTableAction.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/CreateGhostTableAction.java index 8e05cba16b..620d151733 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/CreateGhostTableAction.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/CreateGhostTableAction.java @@ -31,8 +31,8 @@ import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.UnsupportedException; import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor; -import com.oceanbase.odc.service.db.browser.DBObjectOperators; import com.oceanbase.odc.service.db.browser.DBSchemaAccessors; +import com.oceanbase.odc.service.onlineschemachange.OscTableUtil; import com.oceanbase.odc.service.onlineschemachange.ddl.DdlUtils; import com.oceanbase.odc.service.onlineschemachange.fsm.Action; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; @@ -43,7 +43,6 @@ import com.oceanbase.odc.service.onlineschemachange.oscfms.OscActionResult; import com.oceanbase.odc.service.onlineschemachange.oscfms.state.OscStates; import com.oceanbase.tools.dbbrowser.model.DBConstraintType; -import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.dbbrowser.model.DBTableConstraint; @@ -80,18 +79,11 @@ public OscActionResult execute(OscActionContext context) throws Exception { OscStates.CREATE_DATA_TASK.getState()); } - @Override - public void rollback(OscActionContext context) { - log.info("OSC: create table failed rollback, taskID {}", context.getScheduleTask().getId()); - failedOscTask(context); - } - @VisibleForTesting protected void prepareSchema(OnlineSchemaChangeParameters param, OnlineSchemaChangeScheduleTaskParameters taskParam, ConnectionSession session, Long scheduleTaskId, OscActionContext oscContext) throws SQLException { - - dropNewTableIfExits(taskParam, session); - + // drop is required, cause the following sql may failed, and we do retry from the beginning + OscTableUtil.dropNewTableIfExits(taskParam.getDatabaseName(), taskParam.getNewTableNameUnwrapped(), session); SyncJdbcExecutor executor = session.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY); String finalTableDdl; executor.execute(taskParam.getNewTableCreateDdl()); @@ -115,31 +107,6 @@ protected void prepareSchema(OnlineSchemaChangeParameters param, OnlineSchemaCha validateColumnDifferent(taskParam, session); } - @VisibleForTesting - protected void dropNewTableIfExits(OnlineSchemaChangeScheduleTaskParameters taskParam, ConnectionSession session) { - List list = DBSchemaAccessors.create(session) - .showTablesLike(taskParam.getDatabaseName(), taskParam.getNewTableNameUnwrapped()); - // Drop new table suffix with _osc_new_ if exists - if (CollectionUtils.isNotEmpty(list)) { - DBObjectOperators.create(session) - .drop(DBObjectType.TABLE, taskParam.getDatabaseName(), taskParam.getNewTableNameUnwrapped()); - log.info("OSC: drop table {}.{}", taskParam.getDatabaseName(), taskParam.getNewTableNameUnwrapped()); - } - } - - @VisibleForTesting - protected void failedOscTask(OscActionContext context) { - ConnectionSession connectionSession = null; - try { - connectionSession = context.getConnectionProvider().createConnectionSession(); - dropNewTableIfExits(context.getTaskParameter(), connectionSession); - } finally { - if (connectionSession != null) { - connectionSession.expire(); - } - } - } - private void validateColumnDifferent(OnlineSchemaChangeScheduleTaskParameters taskParam, ConnectionSession session) { List originTableColumns = diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsCleanResourcesAction.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsCleanResourcesAction.java index 7ee98a397b..a4b68a01de 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsCleanResourcesAction.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsCleanResourcesAction.java @@ -20,12 +20,15 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.TaskErrorStrategy; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.metadb.schedule.ScheduleEntity; import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; +import com.oceanbase.odc.service.onlineschemachange.OscTableUtil; import com.oceanbase.odc.service.onlineschemachange.exception.OmsException; import com.oceanbase.odc.service.onlineschemachange.fsm.Action; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; @@ -52,7 +55,7 @@ public class OmsCleanResourcesAction implements Action expectedTaskStatus = Lists.newArrayList(TaskStatus.DONE, TaskStatus.FAILED, - TaskStatus.CANCELED, TaskStatus.RUNNING); + TaskStatus.CANCELED, TaskStatus.RUNNING, TaskStatus.ABNORMAL); public OmsCleanResourcesAction(@NonNull OmsProjectOpenApiService omsProjectOpenApiService) { this.projectOpenApiService = omsProjectOpenApiService; @@ -72,9 +75,34 @@ public OscActionResult execute(OscActionContext context) throws Exception { taskParameters.getOmsProjectId(), taskParameters.getUid()); return new OscActionResult(OscStates.CLEAN_RESOURCE.getState(), null, OscStates.CLEAN_RESOURCE.getState()); } + if (!tryDropNewTable(context)) { + // try drop ghost table again + return new OscActionResult(OscStates.CLEAN_RESOURCE.getState(), null, OscStates.CLEAN_RESOURCE.getState()); + } return determinateNextState(scheduleTask, context.getSchedule()); } + protected boolean tryDropNewTable(OscActionContext context) { + ConnectionSession connectionSession = null; + OnlineSchemaChangeScheduleTaskParameters taskParam = context.getTaskParameter(); + String databaseName = taskParam.getDatabaseName(); + String tableName = taskParam.getNewTableNameUnwrapped(); + boolean succeed; + try { + connectionSession = context.getConnectionProvider().createConnectionSession(); + OscTableUtil.dropNewTableIfExits(databaseName, tableName, connectionSession); + succeed = true; + } catch (Throwable e) { + log.warn("osc: drop table = {}.{} failed", databaseName, tableName, e); + succeed = false; + } finally { + if (connectionSession != null) { + connectionSession.expire(); + } + } + return succeed; + } + @VisibleForTesting protected OscActionResult determinateNextState(ScheduleTaskEntity scheduleTask, ScheduleEntity schedule) { Long scheduleId = schedule.getId(); @@ -126,10 +154,10 @@ protected boolean checkAndReleaseProject(OmsProjectControlRequest projectControl try { OmsProjectProgressResponse progressResponse = projectOpenApiService.describeProjectProgress(controlRequest); released = progressResponse.getStatus().isProjectDestroyed(); - } catch (OmsException e) { + } catch (Throwable e) { // representative project has been deleted when message contain "GHANA-PROJECT000001" // or "project is not exists", - if (e.getMessage() != null && e.getMessage().contains("GHANA-PROJECT000001")) { + if (e instanceof OmsException && StringUtils.containsIgnoreCase(e.getMessage(), "GHANA-PROJECT000001")) { released = true; } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java index bc1c3347e8..3c3f0944f0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java @@ -102,7 +102,7 @@ public OscActionResult execute(OscActionContext context) throws Exception { lastResult.getCheckFailedTime(), 25000); }); defaultRenameTableInvoker.invoke(taskParameters, parameters); - // rename table success, jump to clean resoruce state + // rename table success, jump to clean resource state return new OscActionResult(OscStates.SWAP_TABLE.getState(), null, OscStates.CLEAN_RESOURCE.getState()); } finally { if (enableUserMonitor(parameters.getLockUsers())) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java index 37cdf194a1..e5e282d173 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java @@ -110,10 +110,17 @@ public void validate(CreateFlowInstanceReq createReq) { validateSchema(alter.getSchema(), database, connectionConfig.getDialectType()); } } + OscFactoryWrapper oscFactoryWrapper = OscFactoryWrapperGenerator.generate(session.getDialectType()); + TableNameDescriptorFactory tableNameDescriptorFactory = + oscFactoryWrapper.getTableNameDescriptorFactory(); + TableNameDescriptor tableNameDescriptor = tableNameDescriptorFactory.getTableNameDescriptor(tableName); validateTableNameLength(tableName, connectionConfig.getDialectType()); validateOriginTableExists(database, tableName, session); - validateOldTableNotExists(database, tableName, session); + // valid check ghost table and renamed table not exists + validateTableNotExists(database, tableNameDescriptor.getNewTableNameUnWrapped(), session); + validateTableNotExists(database, tableNameDescriptor.getRenamedTableNameUnWrapped(), session); + // valid constraints validateForeignKeyTable(database, tableName, session); validateTableConstraints(database, tableName, session); } @@ -175,14 +182,11 @@ private void validateOriginTableExists(String database, String tableName, Connec () -> CollectionUtils.isNotEmpty(tables)); } - private void validateOldTableNotExists(String database, String tableName, ConnectionSession session) { + private void validateTableNotExists(String database, String tableName, ConnectionSession session) { DBSchemaAccessor accessor = DBSchemaAccessors.create(session); - OscFactoryWrapper oscFactoryWrapper = OscFactoryWrapperGenerator.generate(session.getDialectType()); - TableNameDescriptorFactory tableNameDescriptorFactory = oscFactoryWrapper.getTableNameDescriptorFactory(); - TableNameDescriptor tableNameDescriptor = tableNameDescriptorFactory.getTableNameDescriptor(tableName); - List tables = accessor.showTablesLike(database, tableNameDescriptor.getRenamedTableNameUnWrapped()); + List tables = accessor.showTablesLike(database, DdlUtils.getUnwrappedName(tableName)); PreConditions.validNoDuplicated(ResourceType.OB_TABLE, "tableName", - tableNameDescriptor.getRenamedTableName(), () -> CollectionUtils.isNotEmpty(tables)); + tableName, () -> CollectionUtils.isNotEmpty(tables)); } private void validateForeignKeyTable(String database, String tableName, ConnectionSession session) { From 7e9793b435f5080f2dc39f2391a2b59d578b61ee Mon Sep 17 00:00:00 2001 From: LioRoger Date: Tue, 15 Oct 2024 14:38:59 +0800 Subject: [PATCH 007/118] fix(osc): osc task result not update when oms step info changed (#3678) * fix osc task result not update when oms step changed * fix osc task result not update when oms step changed --- .../OnlineSchemaChangeFlowableTask.java | 81 ++++++++++++++++--- .../OnlineSchemaChangeFlowableTaskTest.java | 44 ++++++++++ 2 files changed, 114 insertions(+), 11 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java index 7b9d314178..00ecbf2e89 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTask.java @@ -16,10 +16,14 @@ package com.oceanbase.odc.service.onlineschemachange; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import javax.validation.constraints.NotNull; + import org.apache.commons.collections4.CollectionUtils; import org.flowable.engine.delegate.DelegateExecution; import org.springframework.beans.factory.annotation.Autowired; @@ -29,6 +33,7 @@ import com.google.common.annotations.VisibleForTesting; import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.shared.constant.FlowStatus; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.core.shared.constant.TaskType; @@ -207,13 +212,14 @@ public void onProgressUpdate(Long taskId, TaskService taskService) { double currentProgressPercentage = getProgressPercentage(scheduleTasks); double prevProgressPercentage = null != previousTaskResult ? getProgressPercentage(previousTaskResult.getTasks()) : 0; - // get swap table flag changed - int currentEnableManualSwapTableFlagCounts = getManualSwapTableEnableFlagCounts(scheduleTasks); - int prevEnableManualSwapTableFlagCounts = - null != previousTaskResult ? getManualSwapTableEnableFlagCounts(previousTaskResult.getTasks()) : 0; + // get swap table flag and oms step changed hint + ScheduleTasksUpdateHint currentHint = getScheduleTasksUpdateHint(scheduleTasks); + ScheduleTasksUpdateHint prevHint = + null != previousTaskResult ? getScheduleTasksUpdateHint(previousTaskResult.getTasks()) + : new ScheduleTasksUpdateHint(0); if (currentProgressPercentage > prevProgressPercentage || dbStatus != currentStatus - || (currentEnableManualSwapTableFlagCounts != prevEnableManualSwapTableFlagCounts)) { + || currentHint.hasDiff(prevHint)) { flowTask.setResultJson(JsonUtils.toJson(new OnlineSchemaChangeTaskResult(scheduleTasks.getContent()))); flowTask.setStatus(currentStatus); flowTask.setProgressPercentage(Math.min(currentProgressPercentage, 100)); @@ -225,22 +231,31 @@ public void onProgressUpdate(Long taskId, TaskService taskService) { } /** - * get counts of tasks with swap table flag enabled + * get schedule task update hint including manual swap table counts and steps */ - protected int getManualSwapTableEnableFlagCounts(Iterable tasks) { + protected ScheduleTasksUpdateHint getScheduleTasksUpdateHint(Iterable tasks) { if (null == tasks) { - return 0; + return new ScheduleTasksUpdateHint(0); } int ret = 0; + Map taskAndStep = new HashMap<>(); for (ScheduleTaskEntity task : tasks) { // check current OnlineSchemaChangeScheduleTaskResult result = JsonUtils.fromJson(task.getResultJson(), OnlineSchemaChangeScheduleTaskResult.class); - if (null != result && result.isManualSwapTableEnabled()) { - ret++; + OnlineSchemaChangeScheduleTaskParameters onlineSchemaChangeScheduleTaskParameters = + JsonUtils.fromJson(task.getParametersJson(), OnlineSchemaChangeScheduleTaskParameters.class); + if (null != result) { + ret += (result.isManualSwapTableEnabled() ? 1 : 0); + taskAndStep.put(task.getId(), result.getCurrentStep()); + } + if (null != onlineSchemaChangeScheduleTaskParameters) { + taskAndStep.compute(task.getId(), (k, v) -> { + return v + onlineSchemaChangeScheduleTaskParameters.getState(); + }); } } - return ret; + return new ScheduleTasksUpdateHint(ret, taskAndStep); } protected double getProgressPercentage(Iterable tasks) { @@ -402,4 +417,48 @@ protected boolean isSuccessful() { protected boolean isFailure() { throw new RuntimeException("not impl"); } + + /** + * hint to determinate if schedule task has changed + */ + protected static final class ScheduleTasksUpdateHint { + private final int enableManualSwapTableFlagCounts; + private final Map taskStepsMap = new HashMap<>(); + + protected ScheduleTasksUpdateHint(int enableManualSwapTableFlagCounts, Map taskStepsMap) { + this.enableManualSwapTableFlagCounts = enableManualSwapTableFlagCounts; + this.taskStepsMap.putAll(taskStepsMap); + } + + protected ScheduleTasksUpdateHint(int enableManualSwapTableFlagCounts) { + this.enableManualSwapTableFlagCounts = enableManualSwapTableFlagCounts; + } + + // if two hint has diff + public boolean hasDiff(@NotNull ScheduleTasksUpdateHint other) { + if (other.enableManualSwapTableFlagCounts != this.enableManualSwapTableFlagCounts) { + return true; + } + if (taskStepsMap.size() != other.taskStepsMap.size()) { + return true; + } + for (Long scheduleTaskId : taskStepsMap.keySet()) { + if (!StringUtils.equalsIgnoreCase(taskStepsMap.get(scheduleTaskId), + other.taskStepsMap.get(scheduleTaskId))) { + return true; + } + } + return false; + } + + @VisibleForTesting + public int getEnableManualSwapTableFlagCounts() { + return enableManualSwapTableFlagCounts; + } + + @VisibleForTesting + public Map getTaskStepsMap() { + return taskStepsMap; + } + } } diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java index ee8e25290a..6f3ecb3870 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java @@ -16,7 +16,10 @@ package com.oceanbase.odc.service.onlineschemachange; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.junit.Assert; import org.junit.Test; @@ -30,6 +33,8 @@ import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.service.flow.task.model.OnlineSchemaChangeTaskResult; +import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeScheduleTaskResult; +import com.oceanbase.odc.service.onlineschemachange.oms.enums.OmsStepName; import com.oceanbase.odc.service.task.TaskService; /** @@ -100,6 +105,45 @@ public void testOnlineSchemaChangeFlowableTaskUpdateProgress() { Assert.assertEquals(onlineSchemaChangeTaskResult.getTasks().get(0).getId().longValue(), 1024); } + @Test + public void testScheduleTasksUpdateHint() { + MockOSCFlowTask mockOSCFlowTask = new MockOSCFlowTask(); + List scheduleTaskEntities = Arrays.asList( + createScheduleTaskEntity(1, true, OmsStepName.FULL_TRANSFER.name()), + createScheduleTaskEntity(2, false, OmsStepName.INCR_TRANSFER.name()), + createScheduleTaskEntity(3, false, null)); + OnlineSchemaChangeFlowableTask.ScheduleTasksUpdateHint scheduleTasksUpdateHint = + mockOSCFlowTask.getScheduleTasksUpdateHint(scheduleTaskEntities); + Assert.assertEquals(scheduleTasksUpdateHint.getEnableManualSwapTableFlagCounts(), 1); + Assert.assertEquals(scheduleTasksUpdateHint.getTaskStepsMap().size(), 2); + Assert.assertEquals(scheduleTasksUpdateHint.getTaskStepsMap().get(1L), OmsStepName.FULL_TRANSFER.name()); + Assert.assertEquals(scheduleTasksUpdateHint.getTaskStepsMap().get(2L), OmsStepName.INCR_TRANSFER.name()); + + Assert.assertFalse(scheduleTasksUpdateHint.hasDiff(scheduleTasksUpdateHint)); + + Assert.assertTrue( + scheduleTasksUpdateHint.hasDiff(new OnlineSchemaChangeFlowableTask.ScheduleTasksUpdateHint(0))); + Assert.assertTrue(scheduleTasksUpdateHint.hasDiff(new OnlineSchemaChangeFlowableTask.ScheduleTasksUpdateHint(1, + Collections.singletonMap(1L, OmsStepName.FULL_TRANSFER.name())))); + Map muted = new HashMap<>(scheduleTasksUpdateHint.getTaskStepsMap()); + muted.put(2L, OmsStepName.APP_SWITCH.name()); + Assert.assertTrue( + scheduleTasksUpdateHint.hasDiff(new OnlineSchemaChangeFlowableTask.ScheduleTasksUpdateHint(1, muted))); + } + + private ScheduleTaskEntity createScheduleTaskEntity(long id, boolean manualSwapTableEnabled, String stepName) { + ScheduleTaskEntity scheduleTask = new ScheduleTaskEntity(); + scheduleTask.setId(id); + OnlineSchemaChangeScheduleTaskResult scheduleTaskResult1 = new OnlineSchemaChangeScheduleTaskResult(); + scheduleTaskResult1.setManualSwapTableEnabled(manualSwapTableEnabled); + scheduleTaskResult1.setCurrentStep(stepName); + if (null != stepName) { + scheduleTask.setResultJson(JsonUtils.toJson(scheduleTaskResult1)); + } + scheduleTask.setStatus(TaskStatus.RUNNING); + return scheduleTask; + } + private static final class MockOSCFlowTask extends OnlineSchemaChangeFlowableTask { private Page createSchedulerTaskEntity() { Page mockPage = Mockito.mock(Page.class); From 29edecb11c7976c02960fe9d8c2432eae885fd2e Mon Sep 17 00:00:00 2001 From: "yh263208@oceanbase.com" Date: Tue, 22 Oct 2024 10:18:50 +0800 Subject: [PATCH 008/118] update submodule --- client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client b/client index b4e114fc69..500782eeb6 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit b4e114fc69e3fe998cb3197ed98877255c077764 +Subproject commit 500782eeb65b5fa2bbb02c2595a499c2b07c49e7 From ce89445c825d83b9656d1df2705f561fe5799d66 Mon Sep 17 00:00:00 2001 From: LioRoger Date: Tue, 22 Oct 2024 10:29:47 +0800 Subject: [PATCH 009/118] feat(task): separate resource state from job (#3700) * task module refactor:separate resource state from job entity * task module refactor:separate resource state from job entity --- script/start-job.sh | 7 +- .../odc/metadb/task/JobRepositoryTest.java | 2 +- .../odc/service/task/EmbedServerTest.java | 2 +- .../service/task/ExecutorIdentifierTest.java | 20 ++ .../odc/service/task/JobSchedulerTest.java | 2 +- .../odc/service/task/NativeK8sClientTest.java | 15 +- .../odc/service/task/ProcessModeTest.java | 2 +- .../odc/service/task/TaskApplicationTest.java | 7 +- .../com/oceanbase/odc/agent/OdcAgent.java} | 27 ++- .../oceanbase/odc/agent}/TaskApplication.java | 15 +- .../odc/agent/runtime}/EmbedServer.java | 4 +- .../runtime}/ExecutorRequestHandler.java | 11 +- .../odc/agent/runtime}/ExitHelper.java | 2 +- .../odc/agent/runtime}/TaskExecutor.java | 5 +- .../odc/agent/runtime}/TaskFactory.java | 6 +- .../runtime}/ThreadPoolTaskExecutor.java | 8 +- .../com/oceanbase/odc/server/OdcServer.java | 13 -- .../web/controller/v2/TaskController.java | 8 +- .../src/main/resources/log4j2-task.xml | 2 +- .../odc/metadb/resource/ResourceEntity.java | 2 +- .../metadb/resource/ResourceRepository.java | 2 +- .../oceanbase/odc/metadb/task/JobEntity.java | 2 +- .../odc/service/common/util/UrlUtils.java | 9 + .../connection/database/DatabaseService.java | 2 +- .../datasecurity/DataMaskingService.java | 4 +- ...tabaseChangeRuntimeFlowableTaskCopied.java | 4 +- .../PreCheckRuntimeFlowableTaskCopied.java | 4 +- ...RollbackPlanRuntimeFlowableTaskCopied.java | 4 +- .../odc/service/git/model/FileChangeType.java | 1 - .../ObjectStorageHandler.java | 2 +- .../util/ObjectStorageUtils.java | 2 +- .../resource/ResourceEntityConverter.java | 4 +- .../odc/service/resource/ResourceID.java | 2 +- .../service/resource/ResourceLocation.java | 2 +- .../odc/service/resource/ResourceManager.java | 13 +- .../service/resource/ResourceOperator.java | 12 +- .../resource/ResourceOperatorBuilder.java | 4 +- .../odc/service/resource/ResourceState.java | 3 + .../BaseNativeK8sResourceOperatorBuilder.java | 2 +- .../service/schedule/job/AbstractDlmJob.java | 2 +- .../job/LogicalDatabaseChangeJob.java | 2 +- .../odc/service/schedule/job/SqlPlanJob.java | 2 +- .../state/StatefulRouteConfiguration.java | 2 +- .../task/{executor/task => }/Task.java | 8 +- .../{executor/task => base}/BaseTask.java | 6 +- .../dataarchive}/DataArchiveTask.java | 3 +- .../databasechange}/DatabaseChangeTask.java | 4 +- .../DatabaseChangeTaskParameters.java | 2 +- .../QuerySensitiveColumnReq.java | 2 +- .../QuerySensitiveColumnResp.java | 2 +- .../LogicalDatabaseChangeTask.java | 3 +- .../precheck}/PreCheckTask.java | 4 +- .../precheck}/PreCheckTaskParameters.java | 2 +- .../rollback}/RollbackPlanTask.java | 4 +- .../rollback}/RollbackPlanTaskParameters.java | 2 +- .../task => base/sqlplan}/SqlPlanTask.java | 3 +- .../service/task/caller/BaseJobCaller.java | 47 +++-- .../caller/DefaultExecutorIdentifier.java | 6 +- .../caller/DefaultK8sJobClientSelector.java | 30 --- .../task/caller/ExecutorIdentifierParser.java | 14 +- .../caller/ExecutorProcessBuilderFactory.java | 11 +- .../odc/service/task/caller/JobCaller.java | 19 +- .../service/task/caller/JobCallerBuilder.java | 32 ++- .../odc/service/task/caller/K8sJobCaller.java | 128 +++++------- .../odc/service/task/caller/K8sJobClient.java | 66 ------ .../service/task/caller/K8sJobResponse.java | 1 - .../task/caller/NativeK8sJobClient.java | 194 ------------------ .../task/caller/NullK8sJobClientSelector.java | 23 --- .../odc/service/task/caller/PodConfig.java | 70 ------- .../odc/service/task/caller/PodStatus.java | 56 ----- .../service/task/caller/ProcessJobCaller.java | 18 +- .../service/task/caller/ResourceIDUtil.java | 90 ++++++++ .../task/config/DefaultJobConfiguration.java | 7 +- .../config/DefaultSpringJobConfiguration.java | 11 +- .../DefaultTaskFrameworkProperties.java | 4 + .../service/task/config/JobConfiguration.java | 9 +- .../service/task/config/K8sProperties.java | 1 - .../config/TaskFrameworkConfiguration.java | 23 --- .../task/config/TaskFrameworkProperties.java | 2 + .../service/task/constants/JobConstants.java | 2 + .../task/dispatch/ImmediateJobDispatcher.java | 29 ++- .../odc/service/task/dummy/DummyTask.java | 68 ++++++ .../task/dummy/LocalMockK8sJobClient.java | 93 +++++++++ .../{task => }/DefaultTaskResult.java | 3 +- .../{task => }/DefaultTaskResultBuilder.java | 3 +- .../{server => }/HeartbeatRequest.java | 2 +- .../executor/{server => }/TaskMonitor.java | 7 +- .../executor/{server => }/TaskReporter.java | 2 +- .../task/executor/{task => }/TaskResult.java | 3 +- .../TraceDecoratorThreadFactory.java | 2 +- .../{server => }/TraceDecoratorUtils.java | 2 +- .../DefaultJobProcessUpdateEvent.java | 2 +- .../DefaultJobProcessUpdateListener.java | 2 +- .../task/processor/DLMResultProcessor.java | 2 +- .../LogicalDBChangeResultProcessor.java | 2 +- .../task/processor/ResultProcessor.java | 2 +- .../DefaultResourceOperatorBuilder.java | 64 +++++- .../k8s => task/resource}/K8sPodResource.java | 2 +- .../resource}/K8sResourceContext.java | 2 +- .../resource}/K8sResourceOperator.java | 8 +- .../resource}/K8sResourceOperatorContext.java | 4 +- .../k8s => task/resource}/PodConfig.java | 2 +- .../client/DefaultK8sJobClientSelector.java | 2 +- .../resource}/client/K8sJobClient.java | 6 +- .../client/K8sJobClientSelector.java | 2 +- .../resource}/client/NativeK8sJobClient.java | 8 +- .../client/NullK8sJobClientSelector.java | 2 +- .../task/schedule/DefaultJobDefinition.java | 2 +- .../DefaultTaskFrameworkDisabledHandler.java | 4 +- .../service/task/schedule/JobDefinition.java | 2 +- .../schedule/MonitorProcessRateLimiter.java | 2 +- .../task/schedule/ResourceDetectUtil.java | 2 +- .../task/schedule/StdJobScheduler.java | 4 +- .../task/schedule/daemon/CheckRunningJob.java | 73 ++++--- ...ecutorJob.java => DestroyResourceJob.java} | 53 ++--- .../task/schedule/daemon/DoCancelingJob.java | 10 + .../schedule/daemon/ResourceManagerUtil.java | 40 ++++ .../schedule/daemon/StartPreparingJob.java | 3 +- .../task/service/ExecutorEndpointManager.java | 23 ++- .../task/service/StdTaskFrameworkService.java | 41 +++- .../task/service/TaskFrameworkService.java | 7 +- .../service/task/util/TaskExecutorClient.java | 5 +- .../resource/k8s/K8sResourceOperatorTest.java | 10 +- .../task/dummy/LocalMockK8sJobClientTest.java | 59 ++++++ 124 files changed, 946 insertions(+), 875 deletions(-) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClientSelector.java => odc-server/src/main/java/com/oceanbase/odc/agent/OdcAgent.java} (52%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/executor => odc-server/src/main/java/com/oceanbase/odc/agent}/TaskApplication.java (92%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server => odc-server/src/main/java/com/oceanbase/odc/agent/runtime}/EmbedServer.java (98%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server => odc-server/src/main/java/com/oceanbase/odc/agent/runtime}/ExecutorRequestHandler.java (93%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server => odc-server/src/main/java/com/oceanbase/odc/agent/runtime}/ExitHelper.java (94%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server => odc-server/src/main/java/com/oceanbase/odc/agent/runtime}/TaskExecutor.java (88%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server => odc-server/src/main/java/com/oceanbase/odc/agent/runtime}/TaskFactory.java (87%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server => odc-server/src/main/java/com/oceanbase/odc/agent/runtime}/ThreadPoolTaskExecutor.java (94%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{task/executor/server => objectstorage}/ObjectStorageHandler.java (98%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{executor/task => }/Task.java (83%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{executor/task => base}/BaseTask.java (97%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{executor/task => base/dataarchive}/DataArchiveTask.java (98%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{runtime => base/databasechange}/DatabaseChangeTask.java (99%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{runtime => base/databasechange}/DatabaseChangeTaskParameters.java (96%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{runtime => base/databasechange}/QuerySensitiveColumnReq.java (95%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{runtime => base/databasechange}/QuerySensitiveColumnResp.java (94%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{executor/task => base/logicdatabasechange}/LogicalDatabaseChangeTask.java (99%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{runtime => base/precheck}/PreCheckTask.java (99%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{runtime => base/precheck}/PreCheckTaskParameters.java (98%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{runtime => base/rollback}/RollbackPlanTask.java (99%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{runtime => base/rollback}/RollbackPlanTaskParameters.java (97%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/{executor/task => base/sqlplan}/SqlPlanTask.java (99%) delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultK8sJobClientSelector.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClient.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NativeK8sJobClient.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NullK8sJobClientSelector.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/PodConfig.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/PodStatus.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ResourceIDUtil.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/DummyTask.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClient.java rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/{task => }/DefaultTaskResult.java (96%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/{task => }/DefaultTaskResultBuilder.java (92%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/{server => }/HeartbeatRequest.java (93%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/{server => }/TaskMonitor.java (97%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/{server => }/TaskReporter.java (97%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/{task => }/TaskResult.java (94%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/{server => }/TraceDecoratorThreadFactory.java (95%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/{server => }/TraceDecoratorUtils.java (94%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{resource/k8s => task/resource}/DefaultResourceOperatorBuilder.java (57%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{resource/k8s => task/resource}/K8sPodResource.java (97%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{resource/k8s => task/resource}/K8sResourceContext.java (97%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{resource/k8s => task/resource}/K8sResourceOperator.java (97%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{resource/k8s => task/resource}/K8sResourceOperatorContext.java (89%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{resource/k8s => task/resource}/PodConfig.java (96%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{resource/k8s => task/resource}/client/DefaultK8sJobClientSelector.java (94%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{resource/k8s => task/resource}/client/K8sJobClient.java (90%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{resource/k8s => task/resource}/client/K8sJobClientSelector.java (93%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{resource/k8s => task/resource}/client/NativeK8sJobClient.java (97%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/{resource/k8s => task/resource}/client/NullK8sJobClientSelector.java (93%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/{DestroyExecutorJob.java => DestroyResourceJob.java} (53%) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/ResourceManagerUtil.java create mode 100644 server/odc-service/src/test/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClientTest.java diff --git a/script/start-job.sh b/script/start-job.sh index 84e7556939..e055e97b6c 100755 --- a/script/start-job.sh +++ b/script/start-job.sh @@ -16,6 +16,9 @@ gc_log_options="-Xloggc:${install_directory}/log/gc.log -XX:+UseGCLogFileRotatio default_heap_options="-XX:MaxRAMPercentage=60.0 -XX:InitialRAMPercentage=60.0" default_gc_options="${gc_basic_options} ${gc_log_options}" default_oom_options="-XX:+ExitOnOutOfMemoryError" +default_agent_main_class_name="com.oceanbase.odc.agent.OdcAgent" +default_spring_boot_loader="org.springframework.boot.loader.PropertiesLauncher" +main_class_caller="-Dloader.main=${default_agent_main_class_name} ${default_spring_boot_loader}" # define some helper functions function usage() { @@ -139,8 +142,8 @@ main() { echo "Starting odc-job..." local cmd="${java_exec} ${remote_debug_options} ${spacev_java_agent_options} ${gc_options} ${heap_options} ${oom_options} - ${extra_options} ${app_options} -jar - ${jar_file} ${app_args}" + ${extra_options} ${app_options} -cp + ${jar_file} ${main_class_caller} ${app_args}" echo "cmd=${cmd}" eval ${cmd} return $? diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java index 1d20c6ada8..817c55baad 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java @@ -21,8 +21,8 @@ import org.springframework.beans.factory.annotation.Autowired; import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTask; import com.oceanbase.odc.service.task.enums.JobStatus; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTask; import com.oceanbase.odc.service.task.util.JobDateUtils; /** diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/EmbedServerTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/EmbedServerTest.java index 8b425de686..ebf5553d9f 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/EmbedServerTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/EmbedServerTest.java @@ -19,7 +19,7 @@ import org.junit.Ignore; import org.junit.Test; -import com.oceanbase.odc.service.task.executor.server.EmbedServer; +import com.oceanbase.odc.agent.runtime.EmbedServer; import lombok.extern.slf4j.Slf4j; diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ExecutorIdentifierTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ExecutorIdentifierTest.java index aaef88bd1b..4adcc832cd 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ExecutorIdentifierTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ExecutorIdentifierTest.java @@ -45,6 +45,26 @@ public void test_parser_successful() throws JobException { Assert.assertEquals(identifier.toString(), identifierString); } + @Test + public void test_parser_decode() { + + // old version1 + String str2 = "http://odc:8989/default/xxx:xxxx001"; + ExecutorIdentifier identifierOld = ExecutorIdentifierParser.parser(str2); + Assert.assertEquals(identifierOld.getNamespace(), "default"); + Assert.assertEquals(identifierOld.getExecutorName(), "xxx:xxxx001"); + // old version2 + String str3 = "http://odc:8989/xxx:xxxx001"; + ExecutorIdentifier identifierOld2 = ExecutorIdentifierParser.parser(str3); + Assert.assertNull(identifierOld2.getNamespace()); + Assert.assertEquals(identifierOld2.getExecutorName(), "xxx:xxxx001"); + // old version3 + String str4 = "http://odc:8989/"; + ExecutorIdentifier identifierOld3 = ExecutorIdentifierParser.parser(str4); + Assert.assertNull(identifierOld3.getNamespace()); + Assert.assertEquals(identifierOld3.getExecutorName(), ""); + } + @Test public void test_executorDefaultValue_successful() throws JobException { ExecutorIdentifier identifier = DefaultExecutorIdentifier.builder().namespace(null) diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/JobSchedulerTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/JobSchedulerTest.java index c34944f72e..e33c3201bc 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/JobSchedulerTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/JobSchedulerTest.java @@ -25,12 +25,12 @@ import com.oceanbase.odc.common.event.LocalEventPublisher; import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTask; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.config.DefaultJobConfiguration; import com.oceanbase.odc.service.task.config.DefaultTaskFrameworkProperties; import com.oceanbase.odc.service.task.dispatch.JobDispatcher; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTask; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobScheduler; import com.oceanbase.odc.service.task.schedule.StdJobScheduler; diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/NativeK8sClientTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/NativeK8sClientTest.java index 5382cd6a3a..3649ec3b96 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/NativeK8sClientTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/NativeK8sClientTest.java @@ -26,12 +26,13 @@ import org.junit.Ignore; import org.junit.Test; -import com.oceanbase.odc.service.task.caller.K8sJobClient; -import com.oceanbase.odc.service.task.caller.K8sJobResponse; -import com.oceanbase.odc.service.task.caller.NativeK8sJobClient; -import com.oceanbase.odc.service.task.caller.PodConfig; import com.oceanbase.odc.service.task.config.K8sProperties; import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.K8sPodResource; +import com.oceanbase.odc.service.task.resource.K8sResourceContext; +import com.oceanbase.odc.service.task.resource.PodConfig; +import com.oceanbase.odc.service.task.resource.client.K8sJobClient; +import com.oceanbase.odc.service.task.resource.client.NativeK8sJobClient; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.util.JobUtils; import com.oceanbase.odc.test.database.TestProperties; @@ -63,10 +64,12 @@ public void test_createJob() throws JobException { String exceptedJobName = JobUtils.generateExecutorName(jobIdentity); List cmd = Arrays.asList("perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"); PodConfig podParam = new PodConfig(); - String generateJobOfName = k8sClient.create("default", exceptedJobName, imageName, cmd, podParam); + K8sResourceContext context = + new K8sResourceContext(podParam, exceptedJobName, imageName, "group", "type", podParam); + K8sPodResource generateJobOfName = k8sClient.create(context); Assert.assertEquals(exceptedJobName, generateJobOfName); - Optional queryJobName = k8sClient.get("default", exceptedJobName); + Optional queryJobName = k8sClient.get("default", exceptedJobName); Assert.assertTrue(queryJobName.isPresent()); Assert.assertEquals(exceptedJobName, queryJobName.get()); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ProcessModeTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ProcessModeTest.java index bb260ddfe9..1eba295e26 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ProcessModeTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ProcessModeTest.java @@ -41,6 +41,7 @@ import com.oceanbase.odc.service.flow.task.model.DatabaseChangeParameters; import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; import com.oceanbase.odc.service.plugin.PluginProperties; +import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTask; import com.oceanbase.odc.service.task.caller.ExecutorProcessBuilderFactory; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.caller.JobEnvironmentFactory; @@ -48,7 +49,6 @@ import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.enums.TaskRunMode; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTask; import com.oceanbase.odc.service.task.schedule.DefaultJobContextBuilder; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobDefinition; diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java index f5f9e85026..1b0ba39479 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java @@ -24,6 +24,8 @@ import org.junit.Test; import com.oceanbase.odc.TestConnectionUtil; +import com.oceanbase.odc.agent.TaskApplication; +import com.oceanbase.odc.agent.runtime.ThreadPoolTaskExecutor; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.shared.constant.ConnectType; @@ -34,15 +36,12 @@ import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.flow.task.model.DatabaseChangeParameters; import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; +import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTask; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.caller.JobEnvironmentFactory; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.enums.TaskRunMode; -import com.oceanbase.odc.service.task.executor.TaskApplication; -import com.oceanbase.odc.service.task.executor.server.ThreadPoolTaskExecutor; -import com.oceanbase.odc.service.task.executor.task.Task; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTask; import com.oceanbase.odc.service.task.schedule.DefaultJobContextBuilder; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobDefinition; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClientSelector.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/OdcAgent.java similarity index 52% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClientSelector.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/OdcAgent.java index 08188b7b48..2003795c22 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClientSelector.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/OdcAgent.java @@ -13,14 +13,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.caller; +package com.oceanbase.odc.agent; + +import com.oceanbase.odc.server.module.Modules; + +import lombok.extern.slf4j.Slf4j; /** - * select the matched K8sJobClient by JobContext.
- * in some deployment scenario, there may exist multiple k8s cluster for job execution. + * main class for Odc agent + * + * @author longpeng.zlp + * @date 2024/8/9 15:31 */ -public interface K8sJobClientSelector { - - K8sJobClient select(JobContext jobContext); +@Slf4j +public class OdcAgent { + public static void main(String[] args) { + log.info("ODC start as task executor mode"); + try { + Modules.load(); + new TaskApplication().run(args); + } catch (Throwable e) { + log.error("Task existed abnormal", e); + } + log.info("Task executor exit."); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskApplication.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/TaskApplication.java similarity index 92% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskApplication.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/TaskApplication.java index 84b5a3758f..f483de2df8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskApplication.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/TaskApplication.java @@ -13,8 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.oceanbase.odc.service.task.executor; +package com.oceanbase.odc.agent; import java.io.File; import java.net.URI; @@ -25,21 +24,21 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LoggerContext; +import com.oceanbase.odc.agent.runtime.EmbedServer; +import com.oceanbase.odc.agent.runtime.ExitHelper; +import com.oceanbase.odc.agent.runtime.TaskFactory; +import com.oceanbase.odc.agent.runtime.ThreadPoolTaskExecutor; import com.oceanbase.odc.common.trace.TaskContextHolder; import com.oceanbase.odc.common.trace.TraceContextHolder; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.caller.JobEnvironmentEncryptor; import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; import com.oceanbase.odc.service.task.executor.context.JobContextProviderFactory; -import com.oceanbase.odc.service.task.executor.server.EmbedServer; -import com.oceanbase.odc.service.task.executor.server.ExitHelper; -import com.oceanbase.odc.service.task.executor.server.TaskFactory; -import com.oceanbase.odc.service.task.executor.server.ThreadPoolTaskExecutor; -import com.oceanbase.odc.service.task.executor.task.BaseTask; import com.oceanbase.odc.service.task.util.JobUtils; import lombok.extern.slf4j.Slf4j; @@ -147,7 +146,7 @@ private void setLog4JConfigXml() { } } - LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false); + LoggerContext context = (LoggerContext) LogManager.getContext(false); // this will force a reconfiguration, MDC context will to take effect context.setConfigLocation(taskLogFile); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/EmbedServer.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/EmbedServer.java similarity index 98% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/EmbedServer.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/EmbedServer.java index 45ff839840..0fe7916461 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/EmbedServer.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/EmbedServer.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.agent.runtime; import java.net.InetSocketAddress; import java.util.concurrent.LinkedBlockingQueue; @@ -28,6 +28,8 @@ import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.service.common.util.UrlUtils; +import com.oceanbase.odc.service.task.executor.TraceDecoratorThreadFactory; +import com.oceanbase.odc.service.task.executor.TraceDecoratorUtils; import com.oceanbase.odc.service.task.util.JobUtils; import io.netty.bootstrap.ServerBootstrap; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExecutorRequestHandler.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java similarity index 93% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExecutorRequestHandler.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java index 70c6289e94..c12daad2c5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExecutorRequestHandler.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.agent.runtime; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -25,14 +25,15 @@ import com.oceanbase.odc.service.common.response.Responses; import com.oceanbase.odc.service.common.response.SuccessResponse; import com.oceanbase.odc.service.common.util.UrlUtils; +import com.oceanbase.odc.service.task.Task; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.constants.JobExecutorUrls; +import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.DefaultTaskResultBuilder; +import com.oceanbase.odc.service.task.executor.TaskMonitor; import com.oceanbase.odc.service.task.executor.logger.LogBiz; import com.oceanbase.odc.service.task.executor.logger.LogBizImpl; import com.oceanbase.odc.service.task.executor.logger.LogUtils; -import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResult; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResultBuilder; -import com.oceanbase.odc.service.task.executor.task.Task; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.util.JobUtils; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExitHelper.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExitHelper.java similarity index 94% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExitHelper.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExitHelper.java index bf80acbf16..e45d121506 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExitHelper.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExitHelper.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.agent.runtime; import java.util.concurrent.CountDownLatch; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskExecutor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java similarity index 88% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskExecutor.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java index efdbc3080a..a749f42a62 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskExecutor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package com.oceanbase.odc.agent.runtime; -package com.oceanbase.odc.service.task.executor.server; - +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.executor.task.BaseTask; import com.oceanbase.odc.service.task.schedule.JobIdentity; /** diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskFactory.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskFactory.java similarity index 87% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskFactory.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskFactory.java index a37376008e..4bb8fbc1a8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskFactory.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskFactory.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.agent.runtime; +import com.oceanbase.odc.service.task.Task; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; -import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.executor.task.Task; /** * @author gaoda.xy diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ThreadPoolTaskExecutor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java similarity index 94% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ThreadPoolTaskExecutor.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java index 45765f174f..9c5c2ef5a9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ThreadPoolTaskExecutor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java @@ -13,8 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.agent.runtime; import java.util.HashMap; import java.util.Map; @@ -28,9 +27,10 @@ import com.oceanbase.odc.common.concurrent.ExecutorUtils; import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.task.TaskThreadFactory; +import com.oceanbase.odc.service.task.Task; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.executor.task.Task; +import com.oceanbase.odc.service.task.executor.TraceDecoratorThreadFactory; import com.oceanbase.odc.service.task.schedule.JobIdentity; import lombok.extern.slf4j.Slf4j; diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/OdcServer.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/OdcServer.java index 142bafce3d..cfdd732833 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/OdcServer.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/OdcServer.java @@ -19,7 +19,6 @@ import java.time.LocalDateTime; import java.util.Map; -import java.util.Objects; import java.util.Properties; import javax.validation.Validation; @@ -45,11 +44,7 @@ import com.oceanbase.odc.core.alarm.AlarmUtils; import com.oceanbase.odc.core.authority.interceptor.MethodAuthorizedPostProcessor; import com.oceanbase.odc.migrate.AbstractMetaDBMigrate; -import com.oceanbase.odc.server.module.Modules; import com.oceanbase.odc.service.config.SystemConfigBootstrap; -import com.oceanbase.odc.service.task.constants.JobConstants; -import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; -import com.oceanbase.odc.service.task.executor.TaskApplication; import lombok.extern.slf4j.Slf4j; @@ -81,14 +76,6 @@ public OdcServer(@Qualifier("metadbMigrate") AbstractMetaDBMigrate metadbMigrate * @param args */ public static void main(String[] args) { - if (Objects.equals(SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_BOOT_MODE), - JobConstants.ODC_BOOT_MODE_EXECUTOR)) { - log.info("ODC start as task executor mode"); - Modules.load(); - new TaskApplication().run(args); - log.info("Task executor exit."); - return; - } AlarmUtils.alarm(SERVER_RESTART, LocalDateTime.now().toString()); initEnv(); System.setProperty("spring.cloud.bootstrap.enabled", "true"); diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java index 57f0b4847a..6c9a44d271 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java @@ -26,10 +26,10 @@ import com.oceanbase.odc.service.common.response.Responses; import com.oceanbase.odc.service.common.response.SuccessResponse; import com.oceanbase.odc.service.datasecurity.DataMaskingService; -import com.oceanbase.odc.service.task.executor.server.HeartbeatRequest; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResult; -import com.oceanbase.odc.service.task.runtime.QuerySensitiveColumnReq; -import com.oceanbase.odc.service.task.runtime.QuerySensitiveColumnResp; +import com.oceanbase.odc.service.task.base.databasechange.QuerySensitiveColumnReq; +import com.oceanbase.odc.service.task.base.databasechange.QuerySensitiveColumnResp; +import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.HeartbeatRequest; import com.oceanbase.odc.service.task.service.TaskFrameworkService; import io.swagger.annotations.ApiOperation; diff --git a/server/odc-server/src/main/resources/log4j2-task.xml b/server/odc-server/src/main/resources/log4j2-task.xml index 4697ceeb3c..8b084ad5e7 100644 --- a/server/odc-server/src/main/resources/log4j2-task.xml +++ b/server/odc-server/src/main/resources/log4j2-task.xml @@ -99,7 +99,7 @@ - + diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java index 6666cf6afe..0c89b3f515 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java @@ -37,7 +37,7 @@ /** * resource table definition for task - * + * * @author longpeng.zlp * @date 2024/8/14 17:37 */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceRepository.java index dbfff2a52e..28faf34665 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceRepository.java @@ -31,7 +31,7 @@ /** * jdbc query for resource_resource table - * + * * @author longpeng.zlp * @date 2024/8/14 17:51 */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/task/JobEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/task/JobEntity.java index a743a85552..27cea4bc61 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/task/JobEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/task/JobEntity.java @@ -78,6 +78,7 @@ public class JobEntity implements Serializable { @Column(name = "execution_times", nullable = false) private Integer executionTimes; + // can parse to resource id @Column(name = "executor_identifier") private String executorIdentifier; @@ -125,5 +126,4 @@ public class JobEntity implements Serializable { @Generated(GenerationTime.ALWAYS) @Column(name = "update_time", insertable = false, updatable = false) private Date updateTime; - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java index 6ffcc99e8b..c158a7de5d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.common.util; import java.net.URLDecoder; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -60,6 +61,14 @@ public static String decode(String encode) { } } + public static String encode(String str) { + try { + return URLEncoder.encode(str, "UTF-8"); + } catch (Exception e) { + return null; + } + } + @Nullable public static String getQueryParameterFirst(String url, String parameterName) { List strings = getQueryParameter(url, parameterName); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index 7b85a031fd..8b5ec20566 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -123,7 +123,7 @@ import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; import com.oceanbase.odc.service.session.model.SqlExecuteResult; -import com.oceanbase.odc.service.task.runtime.PreCheckTaskParameters.AuthorizedDatabase; +import com.oceanbase.odc.service.task.base.precheck.PreCheckTaskParameters.AuthorizedDatabase; import com.oceanbase.tools.dbbrowser.model.DBDatabase; import lombok.NonNull; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/DataMaskingService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/DataMaskingService.java index a7165a624a..a2af9edb97 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/DataMaskingService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/DataMaskingService.java @@ -61,8 +61,8 @@ import com.oceanbase.odc.service.datasecurity.util.MaskingAlgorithmUtil; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.session.model.SqlExecuteResult; -import com.oceanbase.odc.service.task.runtime.QuerySensitiveColumnReq; -import com.oceanbase.odc.service.task.runtime.QuerySensitiveColumnResp; +import com.oceanbase.odc.service.task.base.databasechange.QuerySensitiveColumnReq; +import com.oceanbase.odc.service.task.base.databasechange.QuerySensitiveColumnResp; import com.oceanbase.tools.sqlparser.statement.Statement; import lombok.NonNull; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java index be2f01c11b..d39455e8b6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java @@ -52,12 +52,12 @@ import com.oceanbase.odc.service.objectstorage.model.ObjectMetadata; import com.oceanbase.odc.service.sqlcheck.SqlCheckUtil; import com.oceanbase.odc.service.task.TaskService; +import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTask; +import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTaskParameters; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTask; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTaskParameters; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobDefinition; import com.oceanbase.odc.service.task.schedule.JobScheduler; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java index 25fac88b4c..3338a3aa26 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java @@ -66,11 +66,11 @@ import com.oceanbase.odc.service.sqlcheck.model.CheckResult; import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; import com.oceanbase.odc.service.task.TaskService; +import com.oceanbase.odc.service.task.base.precheck.PreCheckTask; +import com.oceanbase.odc.service.task.base.precheck.PreCheckTaskParameters; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.model.ExecutorInfo; -import com.oceanbase.odc.service.task.runtime.PreCheckTask; -import com.oceanbase.odc.service.task.runtime.PreCheckTaskParameters; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobDefinition; import com.oceanbase.odc.service.task.schedule.JobScheduler; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java index d42a909700..e177c23308 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java @@ -43,10 +43,10 @@ import com.oceanbase.odc.service.objectstorage.model.ObjectMetadata; import com.oceanbase.odc.service.rollbackplan.model.RollbackProperties; import com.oceanbase.odc.service.task.TaskService; +import com.oceanbase.odc.service.task.base.rollback.RollbackPlanTask; +import com.oceanbase.odc.service.task.base.rollback.RollbackPlanTaskParameters; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.enums.JobStatus; -import com.oceanbase.odc.service.task.runtime.RollbackPlanTask; -import com.oceanbase.odc.service.task.runtime.RollbackPlanTaskParameters; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobDefinition; import com.oceanbase.odc.service.task.schedule.JobScheduler; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/git/model/FileChangeType.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/git/model/FileChangeType.java index 830b415b40..eb13618555 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/git/model/FileChangeType.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/git/model/FileChangeType.java @@ -31,5 +31,4 @@ public enum FileChangeType { R, // conflict C - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ObjectStorageHandler.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/ObjectStorageHandler.java similarity index 98% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ObjectStorageHandler.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/ObjectStorageHandler.java index 64ba7681a5..cb92d36e66 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ObjectStorageHandler.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/ObjectStorageHandler.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.service.objectstorage; import java.io.File; import java.io.FileInputStream; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/util/ObjectStorageUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/util/ObjectStorageUtils.java index 7b6a0c230a..e369e375d2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/util/ObjectStorageUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/util/ObjectStorageUtils.java @@ -24,10 +24,10 @@ import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.shared.Verify; import com.oceanbase.odc.service.flow.task.model.SizeAwareInputStream; +import com.oceanbase.odc.service.objectstorage.ObjectStorageHandler; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; import com.oceanbase.odc.service.objectstorage.model.ObjectMetadata; import com.oceanbase.odc.service.objectstorage.model.StorageObject; -import com.oceanbase.odc.service.task.executor.server.ObjectStorageHandler; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceEntityConverter.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceEntityConverter.java index 19f9dc50fd..cf90419a6f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceEntityConverter.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceEntityConverter.java @@ -26,7 +26,7 @@ public interface ResourceEntityConverter { /** * convert resource to ResourceEntity - * + * * @param resource * @return */ @@ -34,7 +34,7 @@ public interface ResourceEntityConverter { /** * convert resourceEntity to resource - * + * * @param resourceEntity * @return */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceID.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceID.java index 22f376831a..96d55b3983 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceID.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceID.java @@ -26,7 +26,7 @@ /** * resource id all resource entity id should contains this class resource id should be impl as * global unique for possible - * + * * @author longpeng.zlp * @date 2024/9/2 16:47 */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceLocation.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceLocation.java index ca23a5a55e..9026823be0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceLocation.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceLocation.java @@ -25,7 +25,7 @@ /** * location of the resource - * + * * @author longpeng.zlp * @date 2024/9/4 10:42 */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java index f080c5cb5f..60e9a3a7d4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java @@ -37,7 +37,6 @@ import com.oceanbase.odc.metadb.resource.ResourceEntity; import com.oceanbase.odc.metadb.resource.ResourceRepository; import com.oceanbase.odc.metadb.resource.ResourceSpecs; -import com.oceanbase.odc.service.resource.k8s.DefaultResourceOperatorBuilder; import com.oceanbase.odc.service.resource.k8s.model.QueryResourceParams; import lombok.NonNull; @@ -45,7 +44,7 @@ /** * resource manager to holds resource allocate and free - * + * * @author longpeng.zlp * @date 2024/8/26 20:17 */ @@ -143,7 +142,7 @@ public Page> list( /** * query resource state with resource id - * + * * @param resourceID * @return * @throws Exception @@ -208,24 +207,24 @@ public void release(@NonNull ResourceID resourceID) { if (!savedResource.isPresent()) { // create resource_resource with DESTROYING state ResourceEntity resourceEntity = new ResourceEntity(); - resourceEntity.setResourceType(DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE); + resourceEntity.setResourceType(resourceID.getType()); resourceEntity.setEndpoint("unknown"); resourceEntity.setCreateTime(new Date(System.currentTimeMillis())); resourceEntity.setRegion(resourceID.getResourceLocation().getRegion()); resourceEntity.setGroupName(resourceID.getResourceLocation().getGroup()); resourceEntity.setNamespace(resourceID.getNamespace()); resourceEntity.setResourceName(resourceID.getIdentifier()); - resourceEntity.setStatus(ResourceState.DESTROYING); + resourceEntity.setStatus(ResourceState.ABANDONED); resourceRepository.save(resourceEntity); } else { // update resource state to destroying - resourceRepository.updateResourceStatus(resourceID, ResourceState.DESTROYING); + resourceRepository.updateResourceStatus(resourceID, ResourceState.ABANDONED); } } /** * real destroy by resource id - * + * * @param resourceID * @return * @throws Exception diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceOperator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceOperator.java index 634fe3f489..c6e7aae393 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceOperator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceOperator.java @@ -22,7 +22,7 @@ /** * operator resource create / recycle / destroy - * + * * @param config of resource creating * @param resource describe * @author longpeng.zlp @@ -32,7 +32,7 @@ public interface ResourceOperator { /** * create a resource by resource context, create may not real create - * + * * @param resourceContext resource config * @return resource, may not available, maybe creating */ @@ -40,7 +40,7 @@ public interface ResourceOperator { /** - * Start current task. This method will be called by {@link TaskExecutor} for fire a task + * Start current task. This method will be called by TaskExecutor for fire a task */ void start(JobContext context); /** - * Stop current task. This method will be called by {@link TaskExecutor} for stop a task + * Stop current task. This method will be called TaskExecutor for stop a task */ boolean stop(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/BaseTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java similarity index 97% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/BaseTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java index 6f3ffc30a8..f68e597a2e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/BaseTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java @@ -13,8 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task.base; import java.util.Collections; import java.util.Map; @@ -24,10 +23,11 @@ import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; +import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.caller.DefaultJobContext; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.enums.JobStatus; -import com.oceanbase.odc.service.task.executor.server.TaskMonitor; +import com.oceanbase.odc.service.task.executor.TaskMonitor; import com.oceanbase.odc.service.task.util.CloudObjectStorageServiceBuilder; import com.oceanbase.odc.service.task.util.JobUtils; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java similarity index 98% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index 491622ade6..868e23e51d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task.base.dataarchive; import java.sql.SQLException; import java.util.ArrayList; @@ -35,6 +35,7 @@ import com.oceanbase.odc.service.dlm.utils.DlmJobIdUtil; import com.oceanbase.odc.service.schedule.job.DLMJobReq; import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.util.JobUtils; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java similarity index 99% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java index d0f000711b..1cb657d068 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.databasechange; import java.io.ByteArrayInputStream; import java.io.File; @@ -91,11 +91,11 @@ import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; import com.oceanbase.odc.service.session.initializer.ConsoleTimeoutInitializer; import com.oceanbase.odc.service.session.model.SqlExecuteResult; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.constants.JobServerUrls; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.executor.task.BaseTask; import com.oceanbase.odc.service.task.util.HttpClientUtils; import com.oceanbase.odc.service.task.util.JobUtils; import com.oceanbase.tools.dbbrowser.parser.ParserUtil; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTaskParameters.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTaskParameters.java similarity index 96% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTaskParameters.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTaskParameters.java index 5fcd10a439..45e1c0671c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTaskParameters.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTaskParameters.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.databasechange; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnReq.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnReq.java similarity index 95% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnReq.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnReq.java index fdd0075481..f049e3b81e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnReq.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnReq.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.databasechange; import java.util.List; import java.util.Set; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnResp.java similarity index 94% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnResp.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnResp.java index 3637f868b4..c443bdf708 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnResp.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.databasechange; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/LogicalDatabaseChangeTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java similarity index 99% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/LogicalDatabaseChangeTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java index 9afb5b40cc..c53f018d09 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/LogicalDatabaseChangeTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task.base.logicdatabasechange; import java.util.ArrayList; import java.util.Collections; @@ -52,6 +52,7 @@ import com.oceanbase.odc.service.connection.logicaldatabase.model.DetailLogicalTableResp; import com.oceanbase.odc.service.schedule.model.PublishLogicalDatabaseChangeReq; import com.oceanbase.odc.service.session.model.SqlExecuteResult; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.tools.dbbrowser.parser.SqlParser; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java similarity index 99% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java index 7990e565e7..9bd28013ef 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.precheck; import java.io.IOException; import java.io.InputStream; @@ -62,9 +62,9 @@ import com.oceanbase.odc.service.sqlcheck.SqlCheckRuleFactory; import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; import com.oceanbase.odc.service.sqlcheck.rule.SqlCheckRules; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; -import com.oceanbase.odc.service.task.executor.task.BaseTask; import com.oceanbase.odc.service.task.util.JobUtils; import lombok.NonNull; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTaskParameters.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTaskParameters.java similarity index 98% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTaskParameters.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTaskParameters.java index bb0dd26324..12971d948a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTaskParameters.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTaskParameters.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.precheck; import java.io.Serializable; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java similarity index 99% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java index e8078a8a2f..897bb15611 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.rollback; import java.io.File; import java.io.IOException; @@ -47,9 +47,9 @@ import com.oceanbase.odc.service.rollbackplan.UnsupportedSqlTypeForRollbackPlanException; import com.oceanbase.odc.service.rollbackplan.model.RollbackPlan; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; -import com.oceanbase.odc.service.task.executor.task.BaseTask; import com.oceanbase.odc.service.task.util.JobUtils; import lombok.extern.slf4j.Slf4j; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTaskParameters.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTaskParameters.java similarity index 97% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTaskParameters.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTaskParameters.java index 3427b84f66..5bb6386e82 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTaskParameters.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTaskParameters.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.rollback; import java.io.Serializable; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/SqlPlanTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java similarity index 99% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/SqlPlanTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java index 02bb9769c1..17c98e28d8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/SqlPlanTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task.base.sqlplan; import java.io.ByteArrayInputStream; import java.io.File; @@ -67,6 +67,7 @@ import com.oceanbase.odc.service.session.initializer.ConsoleTimeoutInitializer; import com.oceanbase.odc.service.session.model.SqlExecuteResult; import com.oceanbase.odc.service.sqlplan.model.SqlPlanTaskResult; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.util.JobUtils; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/BaseJobCaller.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/BaseJobCaller.java index 82e1e863e8..17e285e573 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/BaseJobCaller.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/BaseJobCaller.java @@ -18,6 +18,7 @@ import com.oceanbase.odc.common.event.AbstractEvent; import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.service.resource.ResourceID; import com.oceanbase.odc.service.task.config.JobConfiguration; import com.oceanbase.odc.service.task.config.JobConfigurationHolder; import com.oceanbase.odc.service.task.config.JobConfigurationValidator; @@ -37,7 +38,6 @@ */ @Slf4j public abstract class BaseJobCaller implements JobCaller { - @Override public void start(JobContext context) throws JobException { JobConfigurationValidator.validComponent(); @@ -73,7 +73,7 @@ private void afterStartFailed(JobIdentity ji, ExecutorIdentifier executorIdentifier, Exception ex) throws JobException { if (executorIdentifier != null) { try { - destroy(ji); + finish(ji); } catch (JobException e) { // if destroy failed, domain job will destroy it log.warn("Destroy executor {} occur exception", executorIdentifier); @@ -92,9 +92,11 @@ public void stop(JobIdentity ji) throws JobException { JobEntity jobEntity = taskFrameworkService.find(ji.getId()); String executorEndpoint = jobEntity.getExecutorEndpoint(); + ExecutorIdentifier identifier = ExecutorIdentifierParser.parser(jobEntity.getExecutorIdentifier()); + ResourceID resourceID = ResourceIDUtil.getResourceID(identifier, jobEntity); try { if (executorEndpoint != null - && isExecutorExist(ExecutorIdentifierParser.parser(jobEntity.getExecutorIdentifier()))) { + && isExecutorExist(identifier, resourceID)) { taskExecutorClient.stop(executorEndpoint, ji); } afterStopSucceed(ji); @@ -125,7 +127,7 @@ public void modify(JobIdentity ji, String jobParametersJson) throws JobException } @Override - public void destroy(JobIdentity ji) throws JobException { + public void finish(JobIdentity ji) throws JobException { JobConfigurationValidator.validComponent(); JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); TaskFrameworkService taskFrameworkService = jobConfiguration.getTaskFrameworkService(); @@ -138,12 +140,15 @@ public void destroy(JobIdentity ji) throws JobException { updateExecutorDestroyed(ji); return; } + ExecutorIdentifier identifier = ExecutorIdentifierParser.parser(executorIdentifier); + ResourceID resourceID = ResourceIDUtil.getResourceID(identifier, jobEntity); log.info("Preparing destroy,jobId={}, executorIdentifier={}.", ji.getId(), executorIdentifier); - doDestroy(ji, ExecutorIdentifierParser.parser(executorIdentifier)); + doFinish(ji, identifier, resourceID); } + @Override - public boolean canBeDestroy(JobIdentity ji) { + public boolean canBeFinish(JobIdentity ji) { JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); TaskFrameworkService taskFrameworkService = jobConfiguration.getTaskFrameworkService(); JobEntity jobEntity = taskFrameworkService.find(ji.getId()); @@ -151,24 +156,30 @@ public boolean canBeDestroy(JobIdentity ji) { if (executorIdentifier == null) { return true; } - return canBeDestroy(ji, ExecutorIdentifierParser.parser(executorIdentifier)); + ExecutorIdentifier identifier = ExecutorIdentifierParser.parser(executorIdentifier); + ResourceID resourceID = ResourceIDUtil.getResourceID(identifier, jobEntity); + return canBeFinish(ji, identifier, resourceID); } - protected abstract void doDestroy(JobIdentity ji, ExecutorIdentifier ei) throws JobException; + /** + * detect if job on resource id can be finished + * + * @param ji + * @param ei + * @param resourceID resource id task working on + * @return + */ + protected abstract boolean canBeFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID); + + protected abstract void doFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID) + throws JobException; - protected abstract boolean canBeDestroy(JobIdentity ji, ExecutorIdentifier ei); private void publishEvent(T event) { JobConfiguration configuration = JobConfigurationHolder.getJobConfiguration(); configuration.getEventPublisher().publishEvent(event); } - protected void destroyInternal(ExecutorIdentifier identifier) throws JobException { - if (identifier == null || identifier.getExecutorName() == null) { - return; - } - doDestroyInternal(identifier); - } protected void updateExecutorDestroyed(JobIdentity ji) throws JobException { JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); @@ -185,8 +196,6 @@ protected void updateExecutorDestroyed(JobIdentity ji) throws JobException { protected abstract void doStop(JobIdentity ji) throws JobException; - protected abstract void doDestroyInternal(ExecutorIdentifier identifier) throws JobException; - - protected abstract boolean isExecutorExist(ExecutorIdentifier identifier) throws JobException; - + protected abstract boolean isExecutorExist(ExecutorIdentifier identifier, ResourceID resourceID) + throws JobException; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultExecutorIdentifier.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultExecutorIdentifier.java index 113c492a89..a5f15817cf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultExecutorIdentifier.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultExecutorIdentifier.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.task.caller; import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.service.common.util.UrlUtils; import lombok.Builder; import lombok.Data; @@ -56,11 +57,10 @@ public String toString() { .append(port); if (StringUtils.isNotBlank(namespace)) { sb.append("/"); - sb.append(namespace); + sb.append(UrlUtils.encode(namespace)); } sb.append("/") - .append(executorName); + .append(UrlUtils.encode(executorName)); return sb.toString(); } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultK8sJobClientSelector.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultK8sJobClientSelector.java deleted file mode 100644 index ed460b9814..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultK8sJobClientSelector.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.task.caller; - -public class DefaultK8sJobClientSelector implements K8sJobClientSelector { - - private final K8sJobClient k8sJobClient; - - public DefaultK8sJobClientSelector(K8sJobClient k8sJobClient) { - this.k8sJobClient = k8sJobClient; - } - - @Override - public K8sJobClient select(JobContext jobContext) { - return k8sJobClient; - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorIdentifierParser.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorIdentifierParser.java index 77df720018..b4508105da 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorIdentifierParser.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorIdentifierParser.java @@ -17,6 +17,7 @@ import org.springframework.web.util.UriComponents; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.service.common.util.UrlUtils; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; @@ -35,12 +36,19 @@ public static ExecutorIdentifier parser(String identifierString) { throw new TaskRuntimeException("Illegal executor name : " + path); } - String namespace = path.substring(0, nameIndex).replace("/", ""); + String tmpStr = path.substring(0, nameIndex); + String[] regionAndNamespace = StringUtils.split(tmpStr, "/"); + String namespace = null; + // new version + if (regionAndNamespace.length == 1) { + // old version + namespace = regionAndNamespace[0]; + } return DefaultExecutorIdentifier.builder().host(uriComponents.getHost()) .port(uriComponents.getPort()) .protocol(uriComponents.getScheme()) - .namespace(namespace.length() == 0 ? null : namespace) - .executorName(path.substring(nameIndex).replace("/", "")) + .namespace(StringUtils.isEmpty(namespace) ? null : UrlUtils.decode(namespace)) + .executorName(UrlUtils.decode(path.substring(nameIndex).replace("/", ""))) .build(); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorProcessBuilderFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorProcessBuilderFactory.java index 2c13d54127..2721b03a70 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorProcessBuilderFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorProcessBuilderFactory.java @@ -43,15 +43,18 @@ public ProcessBuilder getProcessBuilder(ProcessConfig processConfig, long jobId, commands.add("-D" + JobUtils.generateExecutorSelectorOnProcess(executorName)); commands.addAll(jvmOptions(processConfig, jobId)); if (ODC_SERVER_EXECUTABLE_JAR.matcher(runtimeMxBean.getClassPath()).matches()) { - // start odc executor by java -jar - commands.add("-jar"); + // start odc executor by java -cp + commands.add("-cp"); // set jar package file name in commands commands.add(runtimeMxBean.getClassPath()); + commands.add("-Dloader.main=" + JobConstants.ODC_AGENT_CLASS_NAME); + commands.add("org.springframework.boot.loader.PropertiesLauncher"); } else { // start odc executor by java -classpath - commands.add("-classpath"); + commands.add("-cp"); commands.add(runtimeMxBean.getClassPath()); - commands.add(JobConstants.ODC_SERVER_CLASS_NAME); + commands.add(JobConstants.ODC_AGENT_CLASS_NAME); + // commands.add("org.springframework.boot.loader.PropertiesLauncher"); } pb.command(commands); pb.directory(new File(".")); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCaller.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCaller.java index 2ada663165..2bdf81e671 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCaller.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCaller.java @@ -52,14 +52,19 @@ public interface JobCaller { */ void modify(JobIdentity ji, String jobParametersJson) throws JobException; - /** - * destroy a odc job executor - * - * @param ji job identity - * @throws JobException throws JobException when stop job failed + * complete the job, process should be quit resource can be released as well + * + * @param ji + * @throws JobException */ - void destroy(JobIdentity ji) throws JobException; + void finish(JobIdentity ji) throws JobException; - boolean canBeDestroy(JobIdentity ji); + /** + * if job can be finished + * + * @param ji + * @return false is job/resource in unknown state + */ + boolean canBeFinish(JobIdentity ji); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCallerBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCallerBuilder.java index bf21acf357..0ffe215036 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCallerBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCallerBuilder.java @@ -22,12 +22,14 @@ import org.apache.commons.io.FileUtils; import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.config.JobConfigurationHolder; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; import com.oceanbase.odc.service.task.enums.TaskMonitorMode; import com.oceanbase.odc.service.task.enums.TaskRunMode; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; +import com.oceanbase.odc.service.task.resource.PodConfig; import com.oceanbase.odc.service.task.util.JobPropertiesUtils; import com.oceanbase.odc.service.task.util.JobUtils; @@ -38,8 +40,14 @@ */ public class JobCallerBuilder { - public static JobCaller buildProcessCaller(JobContext context) { - Map environments = new JobEnvironmentFactory().build(context, TaskRunMode.PROCESS); + /** + * build process caller with given env + * + * @param context + * @param environments env for process builder + * @return + */ + public static JobCaller buildProcessCaller(JobContext context, Map environments) { JobUtils.encryptEnvironments(environments); /** * write JobContext to file in case of exceeding the environments size limit; set the file path in @@ -69,12 +77,15 @@ public static JobCaller buildProcessCaller(JobContext context) { return new ProcessJobCaller(config); } - public static JobCaller buildK8sJobCaller(K8sJobClient k8sJobClient, PodConfig podConfig, JobContext context) { + /** + * build k8s start env + * + * @param context + * @return + */ + public static Map buildK8sEnv(JobContext context) { Map environments = new JobEnvironmentFactory().build(context, TaskRunMode.K8S); - // common environment variables - environments.put(JobEnvKeyConstants.ODC_LOG_DIRECTORY, podConfig.getMountPath()); - Map jobProperties = context.getJobProperties(); // executor listen port @@ -98,11 +109,18 @@ public static JobCaller buildK8sJobCaller(K8sJobClient k8sJobClient, PodConfig p environments.put(JobEnvKeyConstants.ODC_PROPERTY_ENCRYPTION_PREFIX, jasyptProperties.getPrefix()); environments.put(JobEnvKeyConstants.ODC_PROPERTY_ENCRYPTION_SUFFIX, jasyptProperties.getSuffix()); environments.put(JobEnvKeyConstants.ODC_PROPERTY_ENCRYPTION_SALT, jasyptProperties.getSalt()); + return environments; + } + public static JobCaller buildK8sJobCaller(PodConfig podConfig, JobContext context, + ResourceManager resourceManager) { + Map environments = buildK8sEnv(context); + // common environment variables + environments.put(JobEnvKeyConstants.ODC_LOG_DIRECTORY, podConfig.getMountPath()); // do encryption for sensitive information JobUtils.encryptEnvironments(environments); podConfig.setEnvironments(environments); - return new K8sJobCaller(k8sJobClient, podConfig); + return new K8sJobCaller(podConfig, resourceManager); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java index f6a8df5610..d59345e510 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java @@ -16,14 +16,18 @@ package com.oceanbase.odc.service.task.caller; -import static com.oceanbase.odc.service.task.constants.JobConstants.ODC_EXECUTOR_CANNOT_BE_DESTROYED; - import java.util.Optional; -import com.oceanbase.odc.metadb.task.JobEntity; -import com.oceanbase.odc.service.task.config.JobConfiguration; -import com.oceanbase.odc.service.task.config.JobConfigurationHolder; +import com.oceanbase.odc.service.resource.ResourceID; +import com.oceanbase.odc.service.resource.ResourceLocation; +import com.oceanbase.odc.service.resource.ResourceManager; +import com.oceanbase.odc.service.resource.ResourceState; +import com.oceanbase.odc.service.resource.ResourceWithID; import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.DefaultResourceOperatorBuilder; +import com.oceanbase.odc.service.task.resource.K8sPodResource; +import com.oceanbase.odc.service.task.resource.K8sResourceContext; +import com.oceanbase.odc.service.task.resource.PodConfig; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.util.JobUtils; @@ -36,90 +40,72 @@ */ @Slf4j public class K8sJobCaller extends BaseJobCaller { - - private final K8sJobClient client; - private final PodConfig podConfig; - - public K8sJobCaller(K8sJobClient client, PodConfig podConfig) { - this.client = client; - this.podConfig = podConfig; + /** + * base job config + */ + private final PodConfig defaultPodConfig; + private final ResourceManager resourceManager; + + public K8sJobCaller(PodConfig podConfig, ResourceManager resourceManager) { + this.defaultPodConfig = podConfig; + this.resourceManager = resourceManager; } @Override public ExecutorIdentifier doStart(JobContext context) throws JobException { - String jobName = JobUtils.generateExecutorName(context.getJobIdentity()); + try { + ResourceLocation resourceLocation = buildResourceLocation(context); + ResourceWithID resource = + resourceManager.create(resourceLocation, buildK8sResourceContext(context, resourceLocation)); + String arn = resource.getResource().resourceID().getIdentifier(); + return DefaultExecutorIdentifier.builder().namespace(resource.getResource().getNamespace()) + .executorName(arn).build(); + } catch (Throwable e) { + throw new JobException("doStart failed for " + context, e); + } + } - String arn = client.create(podConfig.getNamespace(), jobName, podConfig.getImage(), - podConfig.getCommand(), podConfig); + protected K8sResourceContext buildK8sResourceContext(JobContext context, ResourceLocation resourceLocation) { + String jobName = JobUtils.generateExecutorName(context.getJobIdentity()); + return new K8sResourceContext(defaultPodConfig, jobName, resourceLocation.getRegion(), + resourceLocation.getGroup(), + DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE, context); + } - return DefaultExecutorIdentifier.builder().namespace(podConfig.getNamespace()) - .executorName(arn).build(); + protected ResourceLocation buildResourceLocation(JobContext context) { + // TODO(tianke): confirm is this correct? + String region = ResourceIDUtil.checkAndGetJobProperties(context.getJobProperties(), + ResourceIDUtil.DEFAULT_REGION_PROP_NAME, ResourceIDUtil.DEFAULT_PROP_VALUE); + String group = ResourceIDUtil.checkAndGetJobProperties(context.getJobProperties(), + ResourceIDUtil.DEFAULT_GROUP_PROP_NAME, ResourceIDUtil.DEFAULT_PROP_VALUE); + return new ResourceLocation(region, group); } @Override public void doStop(JobIdentity ji) throws JobException {} @Override - protected void doDestroy(JobIdentity ji, ExecutorIdentifier ei) throws JobException { + protected void doFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID) + throws JobException { + resourceManager.release(resourceID); updateExecutorDestroyed(ji); - Optional k8sJobResponse = client.get(ei.getNamespace(), ei.getExecutorName()); - if (k8sJobResponse.isPresent()) { - if (PodStatus.PENDING == PodStatus.of(k8sJobResponse.get().getResourceStatus())) { - JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); - JobEntity jobEntity = jobConfiguration.getTaskFrameworkService().find(ji.getId()); - if ((System.currentTimeMillis() - jobEntity.getStartedTime().getTime()) / 1000 > podConfig - .getPodPendingTimeoutSeconds()) { - log.info("Pod pending timeout, will be deleted, jobId={}, pod={}, " - + "podPendingTimeoutSeconds={}.", ji.getId(), ei.getExecutorName(), - podConfig.getPodPendingTimeoutSeconds()); - } else { - // Pod cannot be deleted when pod pending is not timeout, - // so throw exception representative delete failed - throw new JobException(ODC_EXECUTOR_CANNOT_BE_DESTROYED + - "Destroy pod failed, jodId={0}, identifier={1}, podStatus={2}", - ji.getId(), ei.getExecutorName(), k8sJobResponse.get().getResourceStatus()); - } - } - log.info("Found pod, delete it, jobId={}, pod={}.", ji.getId(), ei.getExecutorName()); - destroyInternal(ei); - } } @Override - protected boolean canBeDestroy(JobIdentity ji, ExecutorIdentifier ei) { - Optional k8sJobResponse = null; - try { - k8sJobResponse = client.get(ei.getNamespace(), ei.getExecutorName()); - } catch (JobException e) { - log.warn("Get k8s pod occur error, jobId={}", ji.getId(), e); - return false; - } - if (k8sJobResponse.isPresent()) { - if (PodStatus.PENDING == PodStatus.of(k8sJobResponse.get().getResourceStatus())) { - JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); - JobEntity jobEntity = jobConfiguration.getTaskFrameworkService().find(ji.getId()); - if ((System.currentTimeMillis() - jobEntity.getStartedTime().getTime()) / 1000 <= podConfig - .getPodPendingTimeoutSeconds()) { - // Pod cannot be deleted when pod pending is not timeout, - // so throw exception representative cannot delete - log.warn("Cannot destroy pod, pending is not timeout, jodId={}, identifier={}, podStatus={}", - ji.getId(), ei.getExecutorName(), k8sJobResponse.get().getResourceStatus()); - return false; - } - } - } - return true; + protected boolean canBeFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID) { + return resourceManager.canBeDestroyed(resourceID); } @Override - protected void doDestroyInternal(ExecutorIdentifier identifier) throws JobException { - client.delete(podConfig.getNamespace(), identifier.getExecutorName()); - } - - @Override - protected boolean isExecutorExist(ExecutorIdentifier identifier) throws JobException { - Optional executorOptional = client.get(identifier.getNamespace(), identifier.getExecutorName()); - return executorOptional.isPresent() && - PodStatus.of(executorOptional.get().getResourceStatus()) != PodStatus.TERMINATING; + protected boolean isExecutorExist(ExecutorIdentifier identifier, ResourceID resourceID) + throws JobException { + try { + Optional executorOptional = + resourceManager.query(resourceID); + return executorOptional.isPresent() && !ResourceState.isDestroying( + executorOptional.get().getResourceState()); + } catch (Throwable e) { + throw new JobException("invoke isExecutor failed", e); + } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClient.java deleted file mode 100644 index 7ece29a027..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClient.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oceanbase.odc.service.task.caller; - -import java.util.List; -import java.util.Optional; - -import com.oceanbase.odc.service.task.exception.JobException; - -/** - * K8sJobClient is a client to CRUD k8s job in different environment. eg: native or cloud k8s - * - * @author yaobin - * @date 2023-11-15 - * @since 4.2.4 - */ -public interface K8sJobClient { - - /** - * create job in k8s namespace with pod name and use a specific image - * - * @param namespace namespace name - * @param name pod name - * @param image image name - * @param command image start command - * @param podConfig pod config - * @return arn string - * @throws JobException throws exception when create job failed - */ - String create(String namespace, String name, String image, List command, - PodConfig podConfig) throws JobException; - - /** - * get job by serial number in k8s namespace - * - * @param namespace namespace name - * @param arn arn string - * @return job serial number - * @throws JobException throws exception when get job failed - */ - Optional get(String namespace, String arn) throws JobException; - - /** - * delete job by serial number in k8s namespace - * - * @param namespace namespace name - * @param arn arn string - * @return job serial number - * @throws JobException throws exception when delete job failed - */ - String delete(String namespace, String arn) throws JobException; -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobResponse.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobResponse.java index 5507cf4a5b..b85f617c07 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobResponse.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobResponse.java @@ -22,7 +22,6 @@ */ public interface K8sJobResponse { - /** * deploy region */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NativeK8sJobClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NativeK8sJobClient.java deleted file mode 100644 index af86426d70..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NativeK8sJobClient.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oceanbase.odc.service.task.caller; - -import static com.oceanbase.odc.service.task.constants.JobConstants.FIELD_SELECTOR_METADATA_NAME; -import static com.oceanbase.odc.service.task.constants.JobConstants.RESTART_POLICY_NEVER; -import static com.oceanbase.odc.service.task.constants.JobConstants.TEMPLATE_API_VERSION; -import static com.oceanbase.odc.service.task.constants.JobConstants.TEMPLATE_KIND_POD; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import org.apache.commons.collections4.CollectionUtils; - -import com.oceanbase.odc.common.util.EncodeUtils; -import com.oceanbase.odc.common.util.StringUtils; -import com.oceanbase.odc.core.shared.PreConditions; -import com.oceanbase.odc.core.shared.Verify; -import com.oceanbase.odc.core.shared.constant.ErrorCodes; -import com.oceanbase.odc.service.task.config.K8sProperties; -import com.oceanbase.odc.service.task.exception.JobException; - -import io.kubernetes.client.openapi.ApiClient; -import io.kubernetes.client.openapi.ApiException; -import io.kubernetes.client.openapi.Configuration; -import io.kubernetes.client.openapi.apis.CoreV1Api; -import io.kubernetes.client.openapi.models.V1Container; -import io.kubernetes.client.openapi.models.V1EnvVar; -import io.kubernetes.client.openapi.models.V1ObjectMeta; -import io.kubernetes.client.openapi.models.V1Pod; -import io.kubernetes.client.openapi.models.V1PodList; -import io.kubernetes.client.openapi.models.V1PodSpec; -import io.kubernetes.client.util.ClientBuilder; -import io.kubernetes.client.util.Config; -import io.kubernetes.client.util.KubeConfig; -import lombok.NonNull; - -/** - * @author yaobin - * @date 2023-11-15 - * @since 4.2.4 - */ -public class NativeK8sJobClient implements K8sJobClient { - - private final K8sProperties k8sProperties; - private static final long TIMEOUT_MILLS = 60000; - - public NativeK8sJobClient(K8sProperties k8sProperties) throws IOException { - this.k8sProperties = k8sProperties; - ApiClient apiClient = null; - if (StringUtils.isNotBlank(k8sProperties.getKubeConfig())) { - byte[] kubeConfigBytes = EncodeUtils.base64DecodeFromString(k8sProperties.getKubeConfig()); - Verify.notNull(kubeConfigBytes, "kube config"); - try (Reader targetReader = new InputStreamReader(new ByteArrayInputStream(kubeConfigBytes))) { - KubeConfig kubeConfig = KubeConfig.loadKubeConfig(targetReader); - apiClient = ClientBuilder.kubeconfig(kubeConfig).build(); - } - } else if (StringUtils.isNotBlank(k8sProperties.getKubeUrl())) { - apiClient = Config.defaultClient().setBasePath(k8sProperties.getKubeUrl()); - } - Verify.notNull(apiClient, "k8s api client"); - apiClient.setHttpClient(apiClient - .getHttpClient() - .newBuilder() - .readTimeout(TIMEOUT_MILLS, TimeUnit.MILLISECONDS) - .connectTimeout(TIMEOUT_MILLS, TimeUnit.MILLISECONDS) - .pingInterval(1, TimeUnit.MINUTES) - .build()); - Configuration.setDefaultApiClient(apiClient); - } - - @Override - public String create(@NonNull String namespace, @NonNull String name, @NonNull String image, - List command, @NonNull PodConfig podConfig) throws JobException { - validK8sProperties(); - - V1Pod job = getV1Pod(name, image, command, podConfig); - CoreV1Api api = new CoreV1Api(); - try { - V1Pod createdJob = api.createNamespacedPod(namespace, job, null, null, - null, null); - return createdJob.getMetadata().getName(); - } catch (ApiException e) { - if (e.getResponseBody() != null) { - throw new JobException(e.getResponseBody(), e); - } else { - throw new JobException("Create job occur error:", e); - } - } - } - - @Override - public Optional get(@NonNull String namespace, @NonNull String arn) throws JobException { - validK8sProperties(); - CoreV1Api api = new CoreV1Api(); - V1PodList job = null; - try { - job = api.listNamespacedPod(namespace, null, null, null, - FIELD_SELECTOR_METADATA_NAME + "=" + arn, - null, null, null, null, null, null, false); - } catch (ApiException e) { - throw new JobException(e.getResponseBody(), e); - } - Optional v1PodOptional = job.getItems().stream().findAny(); - if (!v1PodOptional.isPresent()) { - return Optional.empty(); - } - V1Pod v1Pod = v1PodOptional.get(); - DefaultK8sJobResponse response = new DefaultK8sJobResponse(); - response.setArn(arn); - response.setName(arn); - response.setRegion(k8sProperties.getRegion()); - response.setResourceStatus(v1Pod.getStatus().getPhase()); - response.setPodIpAddress(v1Pod.getStatus().getPodIP()); - return Optional.of(response); - } - - @Override - public String delete(@NonNull String namespace, @NonNull String arn) throws JobException { - validK8sProperties(); - CoreV1Api api = new CoreV1Api(); - V1Pod pod = null; - try { - pod = api.deleteNamespacedPod(arn, namespace, null, null, - null, null, null, null); - } catch (ApiException e) { - throw new JobException(e.getResponseBody(), e); - } - return pod.getMetadata().getName(); - } - - private V1Pod getV1Pod(String jobName, String image, List command, PodConfig podParam) { - V1Container container = new V1Container() - .name(jobName) - .image(image) - .imagePullPolicy(podParam.getImagePullPolicy()); - - if (CollectionUtils.isNotEmpty(command)) { - container.setCommand(command); - } - - if (podParam.getEnvironments().size() > 0) { - List envVars = podParam.getEnvironments().entrySet().stream() - .map(entry -> new V1EnvVar().name(entry.getKey()).value(entry.getValue())) - .collect(Collectors.toList()); - container.setEnv(envVars); - } - - V1PodSpec v1PodSpec = new V1PodSpec() - .containers(Collections.singletonList(container)) - .restartPolicy(RESTART_POLICY_NEVER); - - return new V1Pod() - .apiVersion(getVersion()).kind(getKind()) - .metadata(new V1ObjectMeta().name(jobName)) - .spec(v1PodSpec); - } - - private void validK8sProperties() { - PreConditions.validArgumentState(k8sProperties.getKubeConfig() != null - || k8sProperties.getKubeUrl() != null, - ErrorCodes.BadArgument, - new Object[] {}, "Target k8s is not set"); - } - - private String getVersion() { - return TEMPLATE_API_VERSION; - } - - private String getKind() { - return TEMPLATE_KIND_POD; - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NullK8sJobClientSelector.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NullK8sJobClientSelector.java deleted file mode 100644 index 06d770ee53..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NullK8sJobClientSelector.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.task.caller; - -public class NullK8sJobClientSelector implements K8sJobClientSelector { - @Override - public K8sJobClient select(JobContext jobContext) { - throw new UnsupportedOperationException("K8sJobClientSelector is not configured."); - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/PodConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/PodConfig.java deleted file mode 100644 index cc39b2482a..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/PodConfig.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oceanbase.odc.service.task.caller; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.oceanbase.odc.service.task.constants.JobConstants; - -import lombok.Data; - -/** - * @author yaobin - * @date 2023-11-16 - * @since 4.2.4 - */ -@Data -public class PodConfig { - - private String region; - - private String namespace; - - private String image; - - private List command; - - private Map environments = new HashMap<>(); - - private Map labels = new HashMap<>(); - - private String imagePullPolicy = JobConstants.IMAGE_PULL_POLICY_ALWAYS; - - private Long podPendingTimeoutSeconds; - - private Double requestCpu; - - private Long requestMem; - - private Double limitCpu; - - private Long limitMem; - - private Boolean enableMount; - - private String mountPath; - - private Long mountDiskSize; - - private Long maxNodeCount; - - private Double nodeCpu; - - private Long nodeMemInMB; -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/PodStatus.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/PodStatus.java deleted file mode 100644 index d1292c739d..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/PodStatus.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.task.caller; - -/** - * @author yaobin - * @date 2024-04-03 - * @since 4.2.4 - */ -public enum PodStatus { - - PENDING("Pending", "INIT"), - RUNNING("Running", "ALLOCATED"), - TERMINATING("Terminating", "PENDING_DELETE"), - UNKNOWN("Known", "UNKNOWN"); - - - private final String name; - private final String alias; - - PodStatus(String name, String alias) { - this.name = name; - this.alias = alias; - } - - public String getName() { - return this.name; - } - - public String getAlias() { - return this.alias; - } - - - public static PodStatus of(String s) { - for (PodStatus status : PodStatus.values()) { - if (status.name().equalsIgnoreCase(s) || status.getAlias().equalsIgnoreCase(s)) { - return status; - } - } - return UNKNOWN; - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java index f8b81fe2e1..090ff3abf4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java @@ -27,6 +27,7 @@ import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.service.common.response.OdcResult; +import com.oceanbase.odc.service.resource.ResourceID; import com.oceanbase.odc.service.task.config.JobConfiguration; import com.oceanbase.odc.service.task.config.JobConfigurationHolder; import com.oceanbase.odc.service.task.enums.JobStatus; @@ -52,7 +53,7 @@ public ProcessJobCaller(ProcessConfig processConfig) { } @Override - protected ExecutorIdentifier doStart(JobContext context) throws JobException { + public ExecutorIdentifier doStart(JobContext context) throws JobException { String executorName = JobUtils.generateExecutorName(context.getJobIdentity()); ProcessBuilder pb = new ExecutorProcessBuilderFactory().getProcessBuilder( @@ -93,14 +94,15 @@ protected ExecutorIdentifier doStart(JobContext context) throws JobException { protected void doStop(JobIdentity ji) throws JobException {} @Override - protected void doDestroy(JobIdentity ji, ExecutorIdentifier ei) throws JobException { - if (isExecutorExist(ei)) { + protected void doFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID) + throws JobException { + if (isExecutorExist(ei, resourceID)) { long pid = Long.parseLong(ei.getNamespace()); log.info("Found process, try kill it, pid={}.", pid); // first update destroy time, second destroy executor. // if executor failed update will be rollback, ensure distributed transaction atomicity. updateExecutorDestroyed(ji); - destroyInternal(ei); + doDestroyInternal(ei); return; } @@ -132,9 +134,8 @@ protected void doDestroy(JobIdentity ji, ExecutorIdentifier ei) throws JobExcept + " may not on this machine, jodId={0}, identifier={1}", ji.getId(), ei); } - @Override - public boolean canBeDestroy(JobIdentity ji, ExecutorIdentifier ei) { - if (isExecutorExist(ei)) { + public boolean canBeFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID) { + if (isExecutorExist(ei, resourceID)) { log.info("Executor be found, jobId={}, identifier={}", ji.getId(), ei); return true; } @@ -152,7 +153,6 @@ public boolean canBeDestroy(JobIdentity ji, ExecutorIdentifier ei) { return false; } - @Override protected void doDestroyInternal(ExecutorIdentifier identifier) throws JobException { long pid = Long.parseLong(identifier.getNamespace()); boolean result = SystemUtils.killProcessByPid(pid); @@ -165,7 +165,7 @@ protected void doDestroyInternal(ExecutorIdentifier identifier) throws JobExcept } @Override - protected boolean isExecutorExist(ExecutorIdentifier identifier) { + protected boolean isExecutorExist(ExecutorIdentifier identifier, ResourceID resourceID) { long pid = Long.parseLong(identifier.getNamespace()); boolean result = SystemUtils.isProcessRunning(pid, JobUtils.generateExecutorSelectorOnProcess(identifier.getExecutorName())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ResourceIDUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ResourceIDUtil.java new file mode 100644 index 0000000000..80db91871e --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ResourceIDUtil.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.caller; + +import java.util.Map; + +import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.service.resource.ResourceID; +import com.oceanbase.odc.service.resource.ResourceLocation; +import com.oceanbase.odc.service.task.resource.DefaultResourceOperatorBuilder; + +import lombok.extern.slf4j.Slf4j; + +/** + * util to help build resource id from job entity and executor identifier + * + * @author longpeng.zlp + * @date 2024/9/2 11:58 + */ +@Slf4j +public class ResourceIDUtil { + public static final String DEFAULT_REGION_PROP_NAME = "region"; + public static final String DEFAULT_GROUP_PROP_NAME = "cloudProvider"; + public static final String DEFAULT_PROP_VALUE = "local"; + + /** + * get with log is missing + * + * @param jobParameters + * @param propName + * @param defaultValue + * @return defaultValue if key is absent in jobParameters + */ + public static String checkAndGetJobProperties(Map jobParameters, String propName, + String defaultValue) { + if (null == jobParameters) { + log.warn("get propName={} failed from job context={} failed, use default value={}", propName, jobParameters, + defaultValue); + return defaultValue; + } + String ret = jobParameters.get(propName); + if (null == ret) { + log.warn("get propName={} failed from job context={} failed, use default value={}", propName, jobParameters, + defaultValue); + return defaultValue; + } else { + return ret; + } + } + + /** + * resolve resourceID with default region and group name + * + * @param executorIdentifier + * @param jobProperties + * @return + */ + public static ResourceID getResourceID(ExecutorIdentifier executorIdentifier, + Map jobProperties) { + String region = checkAndGetJobProperties(jobProperties, DEFAULT_REGION_PROP_NAME, DEFAULT_PROP_VALUE); + String group = checkAndGetJobProperties(jobProperties, DEFAULT_GROUP_PROP_NAME, DEFAULT_PROP_VALUE); + return new ResourceID(new ResourceLocation(region, group), DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE, + executorIdentifier.getNamespace(), + executorIdentifier.getExecutorName()); + } + + /** + * get resource id by jobEntity and executor identifier + * + * @param executorIdentifier + * @param jobEntity + * @return + */ + public static ResourceID getResourceID(ExecutorIdentifier executorIdentifier, JobEntity jobEntity) { + return getResourceID(executorIdentifier, jobEntity.getJobProperties()); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java index 61ebb60f01..3169842dae 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java @@ -22,9 +22,8 @@ import com.oceanbase.odc.service.common.model.HostProperties; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.objectstorage.cloud.model.CloudEnvConfigurations; -import com.oceanbase.odc.service.schedule.ScheduleTaskService; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.TaskService; -import com.oceanbase.odc.service.task.caller.K8sJobClientSelector; import com.oceanbase.odc.service.task.dispatch.JobDispatcher; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; import com.oceanbase.odc.service.task.schedule.JobCredentialProvider; @@ -56,15 +55,13 @@ public abstract class DefaultJobConfiguration implements JobConfiguration { protected TaskService taskService; - protected ScheduleTaskService scheduleTaskService; - protected ConnectionService connectionService; protected JobDispatcher jobDispatcher; protected Scheduler daemonScheduler; - protected K8sJobClientSelector k8sJobClientSelector; + protected ResourceManager resourceManager; protected HostUrlProvider hostUrlProvider; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java index a5638563ea..11745a7253 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java @@ -27,9 +27,8 @@ import com.oceanbase.odc.service.common.model.HostProperties; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.objectstorage.cloud.model.CloudEnvConfigurations; -import com.oceanbase.odc.service.schedule.ScheduleTaskService; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.TaskService; -import com.oceanbase.odc.service.task.caller.K8sJobClientSelector; import com.oceanbase.odc.service.task.dispatch.ImmediateJobDispatcher; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; import com.oceanbase.odc.service.task.schedule.DefaultTaskFrameworkDisabledHandler; @@ -62,9 +61,9 @@ public void afterPropertiesSet() { setJobImageNameProvider(new DefaultJobImageNameProvider(this::getTaskFrameworkProperties)); setConnectionService(ctx.getBean(ConnectionService.class)); setTaskService(ctx.getBean(TaskService.class)); - setScheduleTaskService(ctx.getBean(ScheduleTaskService.class)); setDaemonScheduler((Scheduler) ctx.getBean("taskFrameworkSchedulerFactoryBean")); - setJobDispatcher(new ImmediateJobDispatcher()); + setJobDispatcher(new ImmediateJobDispatcher(ctx.getBean(ResourceManager.class))); + setResourceManager(ctx.getBean(ResourceManager.class)); LocalEventPublisher publisher = new LocalEventPublisher(); TaskFrameworkService tfs = ctx.getBean(TaskFrameworkService.class); if (tfs instanceof StdTaskFrameworkService) { @@ -92,10 +91,6 @@ public TaskFrameworkProperties getTaskFrameworkProperties() { return ctx.getBean(TaskFrameworkProperties.class); } - @Override - public K8sJobClientSelector getK8sJobClientSelector() { - return ctx.getBean(K8sJobClientSelector.class); - } private void initJobRateLimiter() { StartJobRateLimiterSupport limiterSupport = new StartJobRateLimiterSupport(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultTaskFrameworkProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultTaskFrameworkProperties.java index 766abd0386..e6fd835c54 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultTaskFrameworkProperties.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultTaskFrameworkProperties.java @@ -96,4 +96,8 @@ public class DefaultTaskFrameworkProperties implements TaskFrameworkProperties { private String destroyExecutorJobCronExpression; private String pullTaskResultJobCronExpression; + /** + * local k8s debug mode, use process builder mock k8s + */ + private boolean enableK8sLocalDebugMode; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java index 0736c0a235..9ff8d7be81 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java @@ -22,9 +22,8 @@ import com.oceanbase.odc.service.common.model.HostProperties; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.objectstorage.cloud.model.CloudEnvConfigurations; -import com.oceanbase.odc.service.schedule.ScheduleTaskService; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.TaskService; -import com.oceanbase.odc.service.task.caller.K8sJobClientSelector; import com.oceanbase.odc.service.task.dispatch.JobDispatcher; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; import com.oceanbase.odc.service.task.schedule.JobCredentialProvider; @@ -48,9 +47,9 @@ public interface JobConfiguration { CloudEnvConfigurations getCloudEnvConfigurations(); - TaskService getTaskService(); + ResourceManager getResourceManager(); - ScheduleTaskService getScheduleTaskService(); + TaskService getTaskService(); ConnectionService getConnectionService(); @@ -58,8 +57,6 @@ public interface JobConfiguration { JobDispatcher getJobDispatcher(); - K8sJobClientSelector getK8sJobClientSelector(); - HostUrlProvider getHostUrlProvider(); TaskFrameworkService getTaskFrameworkService(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/K8sProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/K8sProperties.java index 64ad181206..e558152f10 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/K8sProperties.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/K8sProperties.java @@ -32,7 +32,6 @@ public class K8sProperties { private String kubeConfig; private String region; private String group; - /** * pod image name with version, odc job will be running in this image */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java index c76dc30999..d35c11087b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java @@ -16,7 +16,6 @@ package com.oceanbase.odc.service.task.config; -import java.io.IOException; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -31,13 +30,8 @@ import org.springframework.scheduling.quartz.SchedulerFactoryBean; import com.oceanbase.odc.common.event.EventPublisher; -import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.service.common.ConditionOnServer; import com.oceanbase.odc.service.objectstorage.cloud.model.CloudEnvConfigurations; -import com.oceanbase.odc.service.task.caller.DefaultK8sJobClientSelector; -import com.oceanbase.odc.service.task.caller.K8sJobClientSelector; -import com.oceanbase.odc.service.task.caller.NativeK8sJobClient; -import com.oceanbase.odc.service.task.caller.NullK8sJobClientSelector; import com.oceanbase.odc.service.task.exception.JobException; import com.oceanbase.odc.service.task.jasypt.DefaultJasyptEncryptorConfigProperties; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; @@ -69,22 +63,6 @@ public JobCredentialProvider jobCredentialProvider(CloudEnvConfigurations cloudE return new DefaultJobCredentialProvider(cloudEnvConfigurations); } - @Lazy - @Bean - @ConditionalOnMissingBean(K8sJobClientSelector.class) - public K8sJobClientSelector k8sJobClientSelector(@Autowired TaskFrameworkProperties taskFrameworkProperties) - throws IOException { - K8sProperties k8sProperties = taskFrameworkProperties.getK8sProperties(); - if (StringUtils.isBlank(k8sProperties.getKubeUrl())) { - log.info("local task k8s cluster is not enabled."); - return new NullK8sJobClientSelector(); - } - log.info("build k8sJobClientSelector, kubeUrl={}, namespace={}", - k8sProperties.getKubeUrl(), k8sProperties.getNamespace()); - NativeK8sJobClient nativeK8sJobClient = new NativeK8sJobClient(k8sProperties); - return new DefaultK8sJobClientSelector(nativeK8sJobClient); - } - @Bean @ConditionalOnBean(TaskFrameworkService.class) public StartJobRateLimiter monitorProcessRateLimiter(@Autowired TaskFrameworkService taskFrameworkService) { @@ -167,5 +145,4 @@ public EventPublisher getEventPublisher() { } }; } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkProperties.java index bda6d91b87..823a27e287 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkProperties.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkProperties.java @@ -72,4 +72,6 @@ public interface TaskFrameworkProperties { String getDestroyExecutorJobCronExpression(); + boolean isEnableK8sLocalDebugMode(); + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/constants/JobConstants.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/constants/JobConstants.java index 3d1dc1690e..2d16737f6a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/constants/JobConstants.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/constants/JobConstants.java @@ -56,6 +56,8 @@ public class JobConstants { public static final String ODC_SERVER_CLASS_NAME = "com.oceanbase.odc.server.OdcServer"; + public static final String ODC_AGENT_CLASS_NAME = "com.oceanbase.odc.agent.OdcAgent"; + public static final String ODC_EXECUTOR_DEFAULT_MOUNT_PATH = "/data/logs/odc-task/runtime"; public static final String ODC_EXECUTOR_PROCESS_PROPERTIES_KEY = "odc.executor"; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dispatch/ImmediateJobDispatcher.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dispatch/ImmediateJobDispatcher.java index a5c54e672d..bb1073383a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dispatch/ImmediateJobDispatcher.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dispatch/ImmediateJobDispatcher.java @@ -22,11 +22,11 @@ import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.caller.JobCaller; import com.oceanbase.odc.service.task.caller.JobCallerBuilder; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.caller.K8sJobClient; -import com.oceanbase.odc.service.task.caller.PodConfig; +import com.oceanbase.odc.service.task.caller.JobEnvironmentFactory; import com.oceanbase.odc.service.task.config.JobConfiguration; import com.oceanbase.odc.service.task.config.JobConfigurationHolder; import com.oceanbase.odc.service.task.config.JobConfigurationValidator; @@ -36,6 +36,7 @@ import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; import com.oceanbase.odc.service.task.enums.TaskRunMode; import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.PodConfig; import com.oceanbase.odc.service.task.schedule.DefaultJobContextBuilder; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.schedule.provider.JobImageNameProvider; @@ -50,6 +51,11 @@ * @since 4.2.4 */ public class ImmediateJobDispatcher implements JobDispatcher { + private final ResourceManager resourceManager; + + public ImmediateJobDispatcher(ResourceManager resourceManager) { + this.resourceManager = resourceManager; + } @Override public void start(JobContext context) throws JobException { @@ -64,21 +70,22 @@ public void stop(JobIdentity ji) throws JobException { } @Override - public void modify(JobIdentity ji, String jobParametersJson) throws JobException { + public void modify(JobIdentity ji, String jobParametersJson) + throws JobException { JobCaller jobCaller = getJobCaller(ji, null); jobCaller.modify(ji, jobParametersJson); } @Override - public void destroy(JobIdentity ji) throws JobException { + public void finish(JobIdentity ji) throws JobException { JobCaller jobCaller = getJobCaller(ji, null); - jobCaller.destroy(ji); + jobCaller.finish(ji); } @Override - public boolean canBeDestroy(JobIdentity ji) { + public boolean canBeFinish(JobIdentity ji) { JobCaller jobCaller = getJobCaller(ji, null); - return jobCaller.canBeDestroy(ji); + return jobCaller.canBeFinish(ji); } private JobCaller getJobCaller(JobIdentity ji, JobContext context) { @@ -92,7 +99,6 @@ private JobCaller getJobCaller(JobIdentity ji, JobContext context) { context = new DefaultJobContextBuilder().build(je); } if (je.getRunMode() == TaskRunMode.K8S) { - K8sJobClient k8sJobClient = config.getK8sJobClientSelector().select(context); PodConfig podConfig = createDefaultPodConfig(config.getTaskFrameworkProperties()); Map labels = JobPropertiesUtils.getLabels(context.getJobProperties()); podConfig.setLabels(labels); @@ -100,9 +106,11 @@ private JobCaller getJobCaller(JobIdentity ji, JobContext context) { if (StringUtils.isNotBlank(regionName)) { podConfig.setRegion(regionName); } - return JobCallerBuilder.buildK8sJobCaller(k8sJobClient, podConfig, context); + return JobCallerBuilder.buildK8sJobCaller(podConfig, context, resourceManager); + } else { + return JobCallerBuilder.buildProcessCaller(context, + new JobEnvironmentFactory().build(context, TaskRunMode.PROCESS)); } - return JobCallerBuilder.buildProcessCaller(context); } private PodConfig createDefaultPodConfig(TaskFrameworkProperties taskFrameworkProperties) { @@ -133,5 +141,4 @@ private PodConfig createDefaultPodConfig(TaskFrameworkProperties taskFrameworkPr podConfig.setPodPendingTimeoutSeconds(k8s.getPodPendingTimeoutSeconds()); return podConfig; } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/DummyTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/DummyTask.java new file mode 100644 index 0000000000..ffffea8a63 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/DummyTask.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.dummy; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.caller.JobContext; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author longpeng.zlp + * @date 2024/8/28 10:51 + */ +@Slf4j +public class DummyTask extends BaseTask { + + private AtomicBoolean stopped = new AtomicBoolean(false); + private AtomicLong loopCount = new AtomicLong(0); + private final long maxLoopCount = 1000000; + + @Override + public double getProgress() { + return (loopCount.get() / maxLoopCount) * 100; + } + + @Override + public String getTaskResult() { + return "has loop count" + loopCount.get(); + } + + @Override + protected void doInit(JobContext context) throws Exception {} + + @Override + protected boolean doStart(JobContext context) throws Exception { + while (!stopped.get() && loopCount.get() < maxLoopCount) { + Thread.sleep(1000); + log.info("dummy task loop for to {}", loopCount.get()); + } + return !stopped.get(); + } + + @Override + protected void doStop() throws Exception { + stopped.set(true); + } + + @Override + protected void doClose() throws Exception { + stopped.set(true); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClient.java new file mode 100644 index 0000000000..3fd2abd342 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClient.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.dummy; + +import java.util.Date; +import java.util.Optional; + +import com.oceanbase.odc.common.util.SystemUtils; +import com.oceanbase.odc.service.resource.ResourceState; +import com.oceanbase.odc.service.task.caller.DefaultExecutorIdentifier; +import com.oceanbase.odc.service.task.caller.DefaultJobContext; +import com.oceanbase.odc.service.task.caller.JobCallerBuilder; +import com.oceanbase.odc.service.task.caller.JobContext; +import com.oceanbase.odc.service.task.caller.ProcessJobCaller; +import com.oceanbase.odc.service.task.caller.ResourceIDUtil; +import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.DefaultResourceOperatorBuilder; +import com.oceanbase.odc.service.task.resource.K8sPodResource; +import com.oceanbase.odc.service.task.resource.K8sResourceContext; +import com.oceanbase.odc.service.task.resource.client.K8sJobClient; +import com.oceanbase.odc.service.task.resource.client.K8sJobClientSelector; +import com.oceanbase.odc.service.task.schedule.JobIdentity; + +/** + * use local process to mock k8s command + * + * @author longpeng.zlp + * @date 2024/8/28 11:18 + */ +public class LocalMockK8sJobClient implements K8sJobClientSelector { + @Override + public K8sJobClient select(String resourceGroup) { + return new LocalProcessClient(); + } + + private static final class LocalProcessClient implements K8sJobClient { + @Override + public K8sPodResource create(K8sResourceContext k8sResourceContext) throws JobException { + JobContext jobContext = getJobContext(k8sResourceContext.getExtraData()); + ProcessJobCaller jobCaller = (ProcessJobCaller) JobCallerBuilder.buildProcessCaller(jobContext, + JobCallerBuilder.buildK8sEnv(jobContext)); + DefaultExecutorIdentifier executorIdentifier = (DefaultExecutorIdentifier) jobCaller.doStart(jobContext); + return new K8sPodResource(k8sResourceContext.getRegion(), k8sResourceContext.getGroup(), + k8sResourceContext.type(), + executorIdentifier.getNamespace(), + executorIdentifier.getExecutorName(), ResourceState.AVAILABLE, + "127.0.0.1:" + executorIdentifier.getPort(), new Date(System.currentTimeMillis())); + } + + private JobContext getJobContext(Object extraData) { + JobContext ret = (JobContext) extraData; + if (null == ret) { + DefaultJobContext jobContext = new DefaultJobContext(); + jobContext.setJobClass(DummyTask.class.getName()); + JobIdentity jobEntity = new JobIdentity(); + jobEntity.setId(1L); + jobContext.setJobIdentity(jobEntity); + ret = jobContext; + } + return ret; + } + + + @Override + public Optional get(String namespace, String arn) throws JobException { + K8sPodResource ret = new K8sPodResource(ResourceIDUtil.DEFAULT_REGION_PROP_NAME, + ResourceIDUtil.DEFAULT_GROUP_PROP_NAME, DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE, + namespace, arn, ResourceState.AVAILABLE, + "127.0.0.1", new Date(System.currentTimeMillis())); + return Optional.of(ret); + } + + @Override + public String delete(String namespace, String arn) throws JobException { + long pid = Long.parseLong(namespace); + SystemUtils.killProcessByPid(pid); + return namespace + ":" + arn; + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResult.java similarity index 96% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResult.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResult.java index 4108a855cb..28bb7a3d5b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResult.java @@ -13,8 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task.executor; import java.util.Map; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResultBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResultBuilder.java similarity index 92% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResultBuilder.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResultBuilder.java index 091d938860..5177922be9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResultBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResultBuilder.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task.executor; import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.util.JobUtils; /** diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/HeartbeatRequest.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/HeartbeatRequest.java similarity index 93% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/HeartbeatRequest.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/HeartbeatRequest.java index b7535bc1f8..57798bc01d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/HeartbeatRequest.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/HeartbeatRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.service.task.executor; import com.oceanbase.odc.service.task.schedule.JobIdentity; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskMonitor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskMonitor.java similarity index 97% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskMonitor.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskMonitor.java index 5f3f091f5d..4a1fe93721 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskMonitor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskMonitor.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.service.task.executor; import java.util.HashMap; import java.util.Map; @@ -29,15 +29,13 @@ import com.oceanbase.odc.common.util.ObjectUtil; import com.oceanbase.odc.core.task.TaskThreadFactory; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.constants.JobAttributeKeyConstants; import com.oceanbase.odc.service.task.constants.JobConstants; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.constants.JobServerUrls; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.executor.logger.LogBizImpl; -import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResult; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResultBuilder; import com.oceanbase.odc.service.task.util.JobUtils; import lombok.extern.slf4j.Slf4j; @@ -213,6 +211,7 @@ private void waitForTaskResultPulled() { } private void uploadLogFileToCloudStorage(DefaultTaskResult finalResult) { + Map logMap = finalResult.getLogMetadata(); if (cloudObjectStorageService != null && cloudObjectStorageService.supported() && JobUtils.isK8sRunModeOfEnv()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskReporter.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskReporter.java similarity index 97% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskReporter.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskReporter.java index 88805089db..448cd52e07 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskReporter.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskReporter.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.service.task.executor; import java.text.MessageFormat; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/TaskResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java similarity index 94% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/TaskResult.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java index 99321bf0e4..d823afd0c4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/TaskResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java @@ -13,8 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task.executor; import java.util.Map; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorThreadFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorThreadFactory.java similarity index 95% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorThreadFactory.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorThreadFactory.java index 26a9ffb6e6..2b73cc40fe 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorThreadFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorThreadFactory.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.service.task.executor; import java.util.concurrent.ThreadFactory; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorUtils.java similarity index 94% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorUtils.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorUtils.java index e300f66d8f..e516918f82 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.service.task.executor; import com.oceanbase.odc.common.trace.TraceDecorator; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateEvent.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateEvent.java index 4bc8adf801..a7cb8ad49a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateEvent.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateEvent.java @@ -16,7 +16,7 @@ package com.oceanbase.odc.service.task.listener; import com.oceanbase.odc.common.event.AbstractEvent; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import lombok.Getter; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java index 4cabb4e8bb..4a353a092c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java @@ -24,7 +24,7 @@ import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.service.schedule.ScheduleTaskService; import com.oceanbase.odc.service.task.TaskService; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.service.TaskFrameworkService; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/DLMResultProcessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/DLMResultProcessor.java index 0bfb1cae6e..c2cc96fe03 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/DLMResultProcessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/DLMResultProcessor.java @@ -26,7 +26,7 @@ import com.oceanbase.odc.service.dlm.DLMService; import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.odc.service.schedule.ScheduleTaskService; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import lombok.extern.slf4j.Slf4j; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/LogicalDBChangeResultProcessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/LogicalDBChangeResultProcessor.java index 8b5385db30..79da5187f1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/LogicalDBChangeResultProcessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/LogicalDBChangeResultProcessor.java @@ -32,7 +32,7 @@ import com.oceanbase.odc.service.connection.logicaldatabase.core.executor.sql.SqlExecutionResultWrapper; import com.oceanbase.odc.service.connection.logicaldatabase.core.model.LogicalDBChangeExecutionUnit; import com.oceanbase.odc.service.schedule.ScheduleTaskService; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import lombok.extern.slf4j.Slf4j; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/ResultProcessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/ResultProcessor.java index 410710aa42..e8e9506af4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/ResultProcessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/ResultProcessor.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.service.task.processor; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; /** * @Author：tinker diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/k8s/DefaultResourceOperatorBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/resource/DefaultResourceOperatorBuilder.java similarity index 57% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/resource/k8s/DefaultResourceOperatorBuilder.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/resource/DefaultResourceOperatorBuilder.java index 8d3afed440..16004d4213 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/k8s/DefaultResourceOperatorBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/resource/DefaultResourceOperatorBuilder.java @@ -13,37 +13,81 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.resource.k8s; +package com.oceanbase.odc.service.task.resource; +import java.io.IOException; import java.util.Optional; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.metadb.resource.ResourceEntity; import com.oceanbase.odc.metadb.resource.ResourceRepository; import com.oceanbase.odc.service.resource.ResourceID; import com.oceanbase.odc.service.resource.ResourceLocation; import com.oceanbase.odc.service.resource.ResourceOperatorBuilder; -import com.oceanbase.odc.service.resource.k8s.client.K8sJobClientSelector; +import com.oceanbase.odc.service.task.config.K8sProperties; +import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; +import com.oceanbase.odc.service.task.dummy.LocalMockK8sJobClient; +import com.oceanbase.odc.service.task.resource.client.DefaultK8sJobClientSelector; +import com.oceanbase.odc.service.task.resource.client.K8sJobClientSelector; +import com.oceanbase.odc.service.task.resource.client.NativeK8sJobClient; +import com.oceanbase.odc.service.task.resource.client.NullK8sJobClientSelector; -import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; /** * default impl for k8s resource operator - * + * * @author longpeng.zlp * @date 2024/9/2 17:33 */ -@AllArgsConstructor +@Component +@Slf4j public class DefaultResourceOperatorBuilder implements ResourceOperatorBuilder { public static final String CLOUD_K8S_POD_TYPE = "cloudK8sPod"; - private final K8sJobClientSelector k8sJobClientSelector; - private final long podPendingTimeoutSeconds; - private final ResourceRepository resourceRepository; + protected K8sJobClientSelector k8sJobClientSelector; + protected K8sProperties k8sProperties; + protected ResourceRepository resourceRepository; + + public DefaultResourceOperatorBuilder(@Autowired TaskFrameworkProperties taskFrameworkProperties, + @Autowired ResourceRepository resourceRepository) throws IOException { + this.k8sProperties = taskFrameworkProperties.getK8sProperties(); + this.resourceRepository = resourceRepository; + this.k8sJobClientSelector = buildK8sJobSelector(taskFrameworkProperties); + } + + /** + * build k8s job selector + */ + protected K8sJobClientSelector buildK8sJobSelector( + TaskFrameworkProperties taskFrameworkProperties) throws IOException { + K8sProperties k8sProperties = taskFrameworkProperties.getK8sProperties(); + K8sJobClientSelector k8sJobClientSelector; + if (taskFrameworkProperties.isEnableK8sLocalDebugMode()) { + // k8s use in local debug mode + log.info("local debug k8s cluster enabled."); + k8sJobClientSelector = new LocalMockK8sJobClient(); + } else if (StringUtils.isBlank(k8sProperties.getKubeUrl())) { + log.info("local task k8s cluster is not enabled."); + k8sJobClientSelector = new NullK8sJobClientSelector(); + } else { + // normal mode + log.info("build k8sJobClientSelector, kubeUrl={}, namespace={}", + k8sProperties.getKubeUrl(), k8sProperties.getNamespace()); + NativeK8sJobClient nativeK8sJobClient = new NativeK8sJobClient(k8sProperties); + k8sJobClientSelector = new DefaultK8sJobClientSelector(nativeK8sJobClient); + } + return k8sJobClientSelector; + } + + @Override public K8sResourceOperator build(ResourceLocation resourceLocation) { return new K8sResourceOperator(new K8sResourceOperatorContext(k8sJobClientSelector, - this::getResourceCreateTimeInSeconds, podPendingTimeoutSeconds)); + this::getResourceCreateTimeInSeconds, k8sProperties.getPodPendingTimeoutSeconds())); } /** @@ -95,7 +139,7 @@ public K8sPodResource toResource(ResourceEntity resourceEntity, Optional jobClass) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/CheckRunningJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/CheckRunningJob.java index 5336d6a43d..4d478d3a79 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/CheckRunningJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/CheckRunningJob.java @@ -24,6 +24,7 @@ import org.springframework.data.domain.Page; import com.oceanbase.odc.common.util.SilentExecutor; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.alarm.AlarmEventNames; import com.oceanbase.odc.core.alarm.AlarmUtils; import com.oceanbase.odc.metadb.task.JobEntity; @@ -76,69 +77,95 @@ private void handleJobRetryingOrFailed(JobEntity jobEntity) { private void doHandleJobRetryingOrFailed(JobEntity jobEntity) { log.info("Start to handle heartbeat timeout job, jobId={}.", jobEntity.getId()); TaskFrameworkService taskFrameworkService = getConfiguration().getTaskFrameworkService(); - JobEntity a = taskFrameworkService.findWithPessimisticLock(jobEntity.getId()); - if (a.getStatus() != JobStatus.RUNNING) { - log.warn("Current job is not RUNNING, abort continue, jobId={}.", a.getId()); + // query from db + JobEntity refreshedJobEntity = taskFrameworkService.findWithPessimisticLock(jobEntity.getId()); + // TODO(lx): confirm this logic why job not in running status not handle + if (refreshedJobEntity.getStatus() != JobStatus.RUNNING) { + log.warn("Current job is not RUNNING, abort continue, jobId={}.", refreshedJobEntity.getId()); return; } - boolean isNeedRetry = checkJobIfRetryNecessary(a); + try { + tryFinishJob(taskFrameworkService, refreshedJobEntity); + } finally { + // mark resource as released to let resource collector collect resource + if (TaskRunMode.K8S == refreshedJobEntity.getRunMode()) { + ResourceManagerUtil.markResourceReleased(refreshedJobEntity, refreshedJobEntity.getExecutorIdentifier(), + getConfiguration().getResourceManager()); + log.info("CheckRunningJob release resource for job = {}", jobEntity); + } + } + } + + private void tryFinishJob(TaskFrameworkService taskFrameworkService, JobEntity jobEntity) { + boolean isNeedRetry = checkJobIfRetryNecessary(jobEntity); if (isNeedRetry) { log.info("Need to restart job, try to set status to RETRYING, jobId={}, oldStatus={}.", - a.getId(), a.getStatus()); + jobEntity.getId(), jobEntity.getStatus()); int rows; - if (TaskRunMode.K8S == a.getRunMode()) { - rows = taskFrameworkService.updateExecutorEndpoint(a.getId(), null); - log.info("Clear executor endpoint why retry task, jobId={}, rows={}", a.getId(), rows); + if (TaskRunMode.K8S == jobEntity.getRunMode()) { + rows = taskFrameworkService.updateExecutorEndpoint(jobEntity.getId(), null); + log.info("Clear executor endpoint why retry task, jobId={}, rows={}", jobEntity.getId(), rows); } rows = taskFrameworkService - .updateStatusDescriptionByIdOldStatus(a.getId(), JobStatus.RUNNING, + .updateStatusDescriptionByIdOldStatus(jobEntity.getId(), JobStatus.RUNNING, JobStatus.RETRYING, "Heart timeout and retrying job"); if (rows > 0) { - log.info("Set job status to RETRYING, jobId={}, oldStatus={}.", a.getId(), a.getStatus()); + log.info("Set job status to RETRYING, jobId={}, oldStatus={}.", jobEntity.getId(), + jobEntity.getStatus()); } else { throw new TaskRuntimeException("Set job status to RETRYING failed, jobId=" + jobEntity.getId()); } } else { log.info("No need to restart job, try to set status to FAILED, jobId={},oldStatus={}.", - a.getId(), a.getStatus()); + jobEntity.getId(), jobEntity.getStatus()); TaskFrameworkProperties taskFrameworkProperties = getConfiguration().getTaskFrameworkProperties(); int rows = taskFrameworkService - .updateStatusToFailedWhenHeartTimeout(a.getId(), + .updateStatusToFailedWhenHeartTimeout(jobEntity.getId(), taskFrameworkProperties.getJobHeartTimeoutSeconds(), "Heart timeout and set job to status FAILED."); if (rows > 0) { - log.info("Set job status to FAILED accomplished, jobId={}, oldStatus={}.", a.getId(), a.getStatus()); + log.info("Set job status to FAILED accomplished, jobId={}, oldStatus={}.", jobEntity.getId(), + jobEntity.getStatus()); AlarmUtils.alarm(AlarmEventNames.TASK_HEARTBEAT_TIMEOUT, - MessageFormat.format("Job running failed due to heart timeout, jobId={0}", a.getId())); + MessageFormat.format("Job running failed due to heart timeout, jobId={0}", jobEntity.getId())); } else { throw new TaskRuntimeException("Set job status to FAILED failed, jobId=" + jobEntity.getId()); } } - if (!getConfiguration().getJobDispatcher().canBeDestroy(JobIdentity.of(a.getId()))) { - log.info("Cannot destroy executor, jobId={}.", a.getId()); + if (!getConfiguration().getJobDispatcher().canBeFinish(JobIdentity.of(jobEntity.getId()))) { + log.info("Cannot destroy executor, jobId={}.", jobEntity.getId()); throw new TaskRuntimeException("Cannot destroy executor, jobId={}" + jobEntity.getId()); } // First try to stop remote job try { - log.info("Try to stop remote job, jobId={}.", a.getId()); - getConfiguration().getJobDispatcher().stop(JobIdentity.of(a.getId())); + log.info("Try to stop remote job, jobId={}.", jobEntity.getId()); + if (StringUtils.isEmpty(jobEntity.getExecutorIdentifier())) { + log.info("found invalid job = {}, resource destroy not confirmed, set status to failed", jobEntity); + taskFrameworkService + .updateStatusDescriptionByIdOldStatus(jobEntity.getId(), JobStatus.RUNNING, + JobStatus.FAILED, "old job not determinate resource has created"); + return; + } + getConfiguration().getJobDispatcher().stop(JobIdentity.of(jobEntity.getId())); } catch (JobException e) { // Process will continue if stop failed and not rollback transaction - log.warn("Try to stop remote failed, jobId={}.", a.getId(), e); + log.warn("Try to stop remote failed, jobId={}.", jobEntity.getId(), e); } - // Second destroy executor + // Second finish job and clean it all try { - log.info("Try to destroy executor, jobId={}.", a.getId()); - getConfiguration().getJobDispatcher().destroy(JobIdentity.of(a.getId())); + log.info("Try to destroy executor, jobId={}.", jobEntity.getId()); + getConfiguration().getJobDispatcher().finish(JobIdentity.of(jobEntity.getId())); } catch (JobException e) { throw new TaskRuntimeException(e); } if (!isNeedRetry) { + // set status to destroyed + taskFrameworkService.updateExecutorToDestroyed(jobEntity.getId()); getConfiguration().getEventPublisher().publishEvent( - new JobTerminateEvent(JobIdentity.of(a.getId()), JobStatus.FAILED)); + new JobTerminateEvent(JobIdentity.of(jobEntity.getId()), JobStatus.FAILED)); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyResourceJob.java similarity index 53% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyResourceJob.java index ce088c96e7..084673b9d6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyResourceJob.java @@ -25,15 +25,15 @@ import com.oceanbase.odc.core.alarm.AlarmEventNames; import com.oceanbase.odc.core.alarm.AlarmUtils; -import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.metadb.resource.ResourceEntity; +import com.oceanbase.odc.service.resource.ResourceID; +import com.oceanbase.odc.service.resource.ResourceLocation; import com.oceanbase.odc.service.task.config.JobConfiguration; import com.oceanbase.odc.service.task.config.JobConfigurationHolder; import com.oceanbase.odc.service.task.config.JobConfigurationValidator; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; import com.oceanbase.odc.service.task.constants.JobConstants; -import com.oceanbase.odc.service.task.exception.JobException; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; -import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.service.TaskFrameworkService; import lombok.extern.slf4j.Slf4j; @@ -45,7 +45,7 @@ */ @Slf4j @DisallowConcurrentExecution -public class DestroyExecutorJob implements Job { +public class DestroyResourceJob implements Job { private JobConfiguration configuration; @@ -57,44 +57,35 @@ public void execute(JobExecutionContext context) throws JobExecutionException { // scan terminate job TaskFrameworkService taskFrameworkService = configuration.getTaskFrameworkService(); TaskFrameworkProperties taskFrameworkProperties = configuration.getTaskFrameworkProperties(); - Page jobs = taskFrameworkService.findTerminalJob(0, + Page resources = taskFrameworkService.findAbandonedResource(0, taskFrameworkProperties.getSingleFetchDestroyExecutorJobRows()); - jobs.forEach(a -> { + resources.forEach(a -> { try { - destroyExecutor(taskFrameworkService, a); + destroyResource(a); } catch (Throwable e) { log.warn("Try to destroy failed, jobId={}.", a.getId(), e); } }); } - private void destroyExecutor(TaskFrameworkService taskFrameworkService, JobEntity jobEntity) { + private void destroyResource(ResourceEntity resourceEntity) { getConfiguration().getTransactionManager().doInTransactionWithoutResult(() -> { - JobEntity lockedEntity = taskFrameworkService.findWithPessimisticLock(jobEntity.getId()); - - if (lockedEntity.getStatus().isTerminated() && lockedEntity.getExecutorIdentifier() != null) { - log.info("Job prepare destroy executor, jobId={},status={}.", lockedEntity.getId(), - lockedEntity.getStatus()); - try { - getConfiguration().getJobDispatcher().destroy(JobIdentity.of(lockedEntity.getId())); - } catch (JobException e) { - log.warn("Destroy executor occur error, jobId={}: ", lockedEntity.getId(), e); - if (e.getMessage() != null && - !e.getMessage().startsWith(JobConstants.ODC_EXECUTOR_CANNOT_BE_DESTROYED)) { - AlarmUtils.alarm(AlarmEventNames.TASK_EXECUTOR_DESTROY_FAILED, - MessageFormat.format("Job executor destroy failed, jobId={0}, message={1}", - lockedEntity.getId(), e.getMessage())); - } - throw new TaskRuntimeException(e); + ResourceID resourceID = new ResourceID(new ResourceLocation(resourceEntity.getRegion(), + resourceEntity.getGroupName()), resourceEntity.getResourceType(), resourceEntity.getNamespace(), + resourceEntity.getResourceName()); + try { + configuration.getResourceManager().destroy(resourceID); + } catch (Throwable e) { + log.warn("DestroyResourceJob destroy resource = {} failed", resourceEntity, e); + if (e.getMessage() != null && + !e.getMessage().startsWith(JobConstants.ODC_EXECUTOR_CANNOT_BE_DESTROYED)) { + AlarmUtils.alarm(AlarmEventNames.TASK_EXECUTOR_DESTROY_FAILED, + MessageFormat.format("Job executor destroy failed, resource={0}, message={1}", + resourceEntity, e.getMessage())); } - log.info("Job destroy executor succeed, jobId={}, status={}.", lockedEntity.getId(), - lockedEntity.getStatus()); - } else if (lockedEntity.getStatus().isTerminated() && lockedEntity.getExecutorIdentifier() == null) { - // It is necessary to update the finish time when the job is terminated but the - // executorIdentifier is null, otherwise, the job cannot be released. - log.info("Executor not found, updating executor to destroyed,jobId={}", lockedEntity.getId()); - taskFrameworkService.updateExecutorToDestroyed(lockedEntity.getId()); + throw new TaskRuntimeException(e); } + log.info("Job destroy resource succeed, resource={}", resourceEntity); }); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java index 5bcefd74bc..e16df67e8e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java @@ -27,6 +27,7 @@ import com.oceanbase.odc.service.task.config.JobConfigurationValidator; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; import com.oceanbase.odc.service.task.enums.JobStatus; +import com.oceanbase.odc.service.task.enums.TaskRunMode; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; import com.oceanbase.odc.service.task.listener.JobTerminateEvent; import com.oceanbase.odc.service.task.schedule.JobIdentity; @@ -73,6 +74,12 @@ private void cancelJob(TaskFrameworkService taskFrameworkService, JobEntity jobE lockedEntity.getId(), lockedEntity.getStatus()); return; } + // mark resource as released + if (TaskRunMode.K8S == lockedEntity.getRunMode()) { + ResourceManagerUtil.markResourceReleased(lockedEntity, lockedEntity.getExecutorIdentifier(), + getConfiguration().getResourceManager()); + log.info("DoCancelingJob release resource for job = {}", jobEntity); + } // For transaction atomic, first update to CANCELED, then stop remote job in executor, // if stop remote failed, transaction will be rollback int rows = getConfiguration().getTaskFrameworkService() @@ -91,6 +98,9 @@ private void cancelJob(TaskFrameworkService taskFrameworkService, JobEntity jobE // MessageFormat.format("Cancel job failed, jobId={0}", lockedEntity.getId())); // throw new TaskRuntimeException(e); // } + // set status to destroyed + // TODO(lx): do finish action before release resource + taskFrameworkService.updateExecutorToDestroyed(jobEntity.getId()); log.info("Job be cancelled successfully, jobId={}, oldStatus={}.", lockedEntity.getId(), lockedEntity.getStatus()); getConfiguration().getEventPublisher().publishEvent( diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/ResourceManagerUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/ResourceManagerUtil.java new file mode 100644 index 0000000000..7fed362876 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/ResourceManagerUtil.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.schedule.daemon; + +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.service.resource.ResourceID; +import com.oceanbase.odc.service.resource.ResourceManager; +import com.oceanbase.odc.service.task.caller.ExecutorIdentifier; +import com.oceanbase.odc.service.task.caller.ExecutorIdentifierParser; +import com.oceanbase.odc.service.task.caller.ResourceIDUtil; + +/** + * @author longpeng.zlp + * @date 2024/8/15 17:54 + */ +public class ResourceManagerUtil { + public static void markResourceReleased(JobEntity jobEntity, String executorIdentifierStr, + ResourceManager resourceManager) { + // mark resource as released to let resource collector collect resource + if (StringUtils.isNotEmpty(executorIdentifierStr)) { + ExecutorIdentifier executorIdentifier = ExecutorIdentifierParser.parser(executorIdentifierStr); + ResourceID resourceID = ResourceIDUtil.getResourceID(executorIdentifier, jobEntity); + resourceManager.release(resourceID); + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java index ed4e03bcb0..952c063dfc 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java @@ -65,7 +65,8 @@ public void execute(JobExecutionContext context) throws JobExecutionException { configuration.getTaskFrameworkDisabledHandler().handleJobToFailed(); return; } - if (!ResourceDetectUtil.isResourceAvailable(configuration.getTaskFrameworkProperties())) { + // check if local compute resource is enough for process mode + if (!ResourceDetectUtil.isProcessResourceAvailable(configuration.getTaskFrameworkProperties())) { return; } TaskFrameworkProperties taskFrameworkProperties = configuration.getTaskFrameworkProperties(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/ExecutorEndpointManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/ExecutorEndpointManager.java index 61c9453d80..24d03e3cbf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/ExecutorEndpointManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/ExecutorEndpointManager.java @@ -26,14 +26,15 @@ import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.service.cloud.model.CloudProvider; +import com.oceanbase.odc.service.resource.ResourceID; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.caller.ExecutorIdentifier; import com.oceanbase.odc.service.task.caller.ExecutorIdentifierParser; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.caller.K8sJobClient; -import com.oceanbase.odc.service.task.caller.K8sJobClientSelector; -import com.oceanbase.odc.service.task.caller.K8sJobResponse; +import com.oceanbase.odc.service.task.caller.ResourceIDUtil; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.enums.TaskRunMode; +import com.oceanbase.odc.service.task.resource.K8sPodResource; import com.oceanbase.odc.service.task.schedule.DefaultJobContextBuilder; import com.oceanbase.odc.service.task.util.JobPropertiesUtils; @@ -44,10 +45,10 @@ @Service @SkipAuthorize("odc internal usage") public class ExecutorEndpointManager { - @Autowired - private K8sJobClientSelector k8sJobClientSelector; @Autowired private TaskFrameworkService taskFrameworkService; + @Autowired + private ResourceManager resourceManager; private ExecutorHostAdapter hostAdapter = null; @@ -72,7 +73,6 @@ public String getExecutorEndpoint(@NonNull JobEntity je) { // here TaskRunMode.K8S == je.getRunMode() JobContext jobContext = new DefaultJobContextBuilder().build(je); ExecutorIdentifier executorIdentifier = ExecutorIdentifierParser.parser(je.getExecutorIdentifier()); - K8sJobClient k8sJobClient = k8sJobClientSelector.select(jobContext); Map jobProperties = jobContext.getJobProperties(); int executorListenPort = JobPropertiesUtils.getExecutorListenPort(jobProperties); if (executorListenPort <= 0) { @@ -80,10 +80,11 @@ public String getExecutorEndpoint(@NonNull JobEntity je) { + ", executorListenPort=" + executorListenPort); } try { - Optional responseOptional = k8sJobClient.get(executorIdentifier.getNamespace(), - executorIdentifier.getExecutorName()); - if (responseOptional.isPresent()) { - K8sJobResponse response = responseOptional.get(); + ResourceID resourceID = ResourceIDUtil.getResourceID(executorIdentifier, je); + Optional resourceOptional = + resourceManager.query(resourceID); + if (resourceOptional.isPresent()) { + K8sPodResource response = resourceOptional.get(); String podIpAddress = response.getPodIpAddress(); if (StringUtils.isNotBlank(podIpAddress)) { String adaptedHost = adaptHost(podIpAddress, jobProperties); @@ -92,7 +93,7 @@ public String getExecutorEndpoint(@NonNull JobEntity je) { return executorEndpoint; } else { throw new RuntimeException( - "Failed to get executor endpoint, pod status=" + response.getResourceStatus()); + "Failed to get executor endpoint, pod status=" + response.getResourceState()); } } else { throw new RuntimeException("Failed to get executor endpoint, pod not exists"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java index 625c4d2171..1636e30be2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java @@ -59,19 +59,27 @@ import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.NotFoundException; +import com.oceanbase.odc.metadb.resource.ResourceEntity; +import com.oceanbase.odc.metadb.resource.ResourceRepository; import com.oceanbase.odc.metadb.task.JobAttributeEntity; import com.oceanbase.odc.metadb.task.JobAttributeRepository; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.metadb.task.JobRepository; +import com.oceanbase.odc.service.resource.ResourceID; +import com.oceanbase.odc.service.resource.ResourceManager; +import com.oceanbase.odc.service.resource.ResourceState; +import com.oceanbase.odc.service.task.caller.ExecutorIdentifier; +import com.oceanbase.odc.service.task.caller.ExecutorIdentifierParser; +import com.oceanbase.odc.service.task.caller.ResourceIDUtil; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; import com.oceanbase.odc.service.task.constants.JobAttributeEntityColumn; import com.oceanbase.odc.service.task.constants.JobEntityColumn; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.enums.TaskRunMode; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.executor.server.HeartbeatRequest; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResult; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.HeartbeatRequest; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.listener.DefaultJobProcessUpdateEvent; import com.oceanbase.odc.service.task.listener.JobTerminateEvent; import com.oceanbase.odc.service.task.processor.DLMResultProcessor; @@ -102,6 +110,10 @@ public class StdTaskFrameworkService implements TaskFrameworkService { @Autowired private JobRepository jobRepository; @Autowired + private ResourceRepository resourceRepository; + @Autowired + private ResourceManager resourceManager; + @Autowired private JobAttributeRepository jobAttributeRepository; @Setter private EventPublisher publisher; @@ -169,6 +181,15 @@ public Page findTerminalJob(int page, int size) { return page(condition, page, size); } + @Override + public Page findAbandonedResource(int page, int size) { + Specification specification = SpecificationUtil.columnLate(ResourceEntity.CREATE_TIME, + JobDateUtils.getCurrentDateSubtractDays(RECENT_DAY)); + Specification condition = Specification.where(specification) + .and(SpecificationUtil.columnIn(ResourceEntity.STATUS, Lists.newArrayList(ResourceState.ABANDONED))); + return resourceRepository.findAll(condition, PageRequest.of(page, size)); + } + @Override public Page findHeartTimeTimeoutJobs(int timeoutSeconds, int page, int size) { Specification condition = Specification.where(getRecentDaySpec(RECENT_DAY)) @@ -310,8 +331,8 @@ public void handleResult(TaskResult taskResult) { log.warn("Job identity is not exists by id {}", taskResult.getJobIdentity().getId()); return; } + // that's may be a dangerous operation if task report too frequent saveOrUpdateLogMetadata(taskResult, je.getId(), je.getStatus()); - if (je.getStatus().isTerminated() || je.getStatus() == JobStatus.CANCELING) { log.warn("Job is finished, ignore result, jobId={}, currentStatus={}", je.getId(), je.getStatus()); return; @@ -326,6 +347,7 @@ public void handleResult(TaskResult taskResult) { } // TODO: update task entity only when progress changed int rows = updateTaskResult(taskResult, je); + tryReleaseResource(je, taskResult.getStatus().isTerminated()); if (rows > 0) { taskResultPublisherExecutor .execute(() -> publisher.publishEvent(new DefaultJobProcessUpdateEvent(taskResult))); @@ -422,6 +444,8 @@ private void doRefreshResult(Long id) throws JobException { } int rows = updateTaskResult(result, je); + // release resource + tryReleaseResource(je, result.getStatus().isTerminated()); if (rows == 0) { log.warn("Update task result failed, the job may finished or deleted already, jobId={}", id); return; @@ -442,6 +466,15 @@ private void doRefreshResult(Long id) throws JobException { } } + protected void tryReleaseResource(JobEntity jobEntity, boolean isJobDone) { + // release resource + if (isJobDone && TaskRunMode.K8S == jobEntity.getRunMode()) { + ExecutorIdentifier executorIdentifier = ExecutorIdentifierParser.parser(jobEntity.getExecutorIdentifier()); + ResourceID resourceID = ResourceIDUtil.getResourceID(executorIdentifier, jobEntity); + resourceManager.release(resourceID); + } + } + private boolean updateHeartbeatTime(Long id) { int rows = jobRepository.updateHeartbeatTime(id, JobStatus.RUNNING); if (rows > 0) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/TaskFrameworkService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/TaskFrameworkService.java index 7f149b955d..f7973f02b9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/TaskFrameworkService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/TaskFrameworkService.java @@ -22,11 +22,12 @@ import org.springframework.data.domain.Page; +import com.oceanbase.odc.metadb.resource.ResourceEntity; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.enums.TaskRunMode; -import com.oceanbase.odc.service.task.executor.server.HeartbeatRequest; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.HeartbeatRequest; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.schedule.JobDefinition; /** @@ -52,6 +53,8 @@ public interface TaskFrameworkService { Page findTerminalJob(int page, int size); + Page findAbandonedResource(int page, int size); + JobEntity findWithPessimisticLock(Long id); Page find(JobStatus status, int page, int size); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java index 5a8f1e2c38..3e375a6728 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java @@ -28,8 +28,7 @@ import com.oceanbase.odc.service.schedule.ScheduleLogProperties; import com.oceanbase.odc.service.task.constants.JobExecutorUrls; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.executor.server.ExecutorRequestHandler; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.DefaultTaskResult; import com.oceanbase.odc.service.task.model.OdcTaskLogLevel; import com.oceanbase.odc.service.task.schedule.JobIdentity; @@ -38,7 +37,7 @@ /** * task-executor api calling encapsulation
- * see @{@link JobExecutorUrls} @{@link ExecutorRequestHandler} + * see @{@link JobExecutorUrls} */ @Slf4j @Component diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/resource/k8s/K8sResourceOperatorTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/resource/k8s/K8sResourceOperatorTest.java index cecbedd889..d71e1831b1 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/resource/k8s/K8sResourceOperatorTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/resource/k8s/K8sResourceOperatorTest.java @@ -25,9 +25,15 @@ import com.oceanbase.odc.service.resource.ResourceID; import com.oceanbase.odc.service.resource.ResourceLocation; import com.oceanbase.odc.service.resource.ResourceState; -import com.oceanbase.odc.service.resource.k8s.client.K8sJobClient; -import com.oceanbase.odc.service.resource.k8s.client.K8sJobClientSelector; import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.DefaultResourceOperatorBuilder; +import com.oceanbase.odc.service.task.resource.K8sPodResource; +import com.oceanbase.odc.service.task.resource.K8sResourceContext; +import com.oceanbase.odc.service.task.resource.K8sResourceOperator; +import com.oceanbase.odc.service.task.resource.K8sResourceOperatorContext; +import com.oceanbase.odc.service.task.resource.PodConfig; +import com.oceanbase.odc.service.task.resource.client.K8sJobClient; +import com.oceanbase.odc.service.task.resource.client.K8sJobClientSelector; /** * test for K8SResourceOperator diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClientTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClientTest.java new file mode 100644 index 0000000000..cecc9eccd9 --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClientTest.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.dummy; + +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; +import org.mockito.Mockito; + +import com.oceanbase.odc.service.common.model.HostProperties; +import com.oceanbase.odc.service.task.config.JobConfiguration; +import com.oceanbase.odc.service.task.config.JobConfigurationHolder; +import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; +import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.K8sPodResource; +import com.oceanbase.odc.service.task.resource.K8sResourceContext; +import com.oceanbase.odc.service.task.resource.PodConfig; +import com.oceanbase.odc.service.task.resource.client.K8sJobClient; +import com.oceanbase.odc.service.task.schedule.JobCredentialProvider; + +/** + * @author longpeng.zlp + * @date 2024/8/28 14:27 + */ +public class LocalMockK8sJobClientTest { + @Ignore + @Test + public void testLocalProcessStart() throws JobException { + JobConfiguration jobConfiguration = Mockito.mock(JobConfiguration.class); + JobCredentialProvider jobCredentialProvider = Mockito.mock(JobCredentialProvider.class); + TaskFrameworkProperties taskFrameworkProperties = Mockito.mock(TaskFrameworkProperties.class); + Mockito.when(taskFrameworkProperties.getJobProcessMaxMemorySizeInMB()).thenReturn(2048L); + Mockito.when(taskFrameworkProperties.getJobProcessMinMemorySizeInMB()).thenReturn(2048L); + HostProperties hostProperties = Mockito.mock(HostProperties.class); + Mockito.when(jobConfiguration.getJobCredentialProvider()).thenReturn(jobCredentialProvider); + Mockito.when(jobConfiguration.getTaskFrameworkProperties()).thenReturn(taskFrameworkProperties); + Mockito.when(jobConfiguration.getHostProperties()).thenReturn(hostProperties); + JobConfigurationHolder.setJobConfiguration(jobConfiguration); + LocalMockK8sJobClient localMockK8sJobClient = new LocalMockK8sJobClient(); + K8sJobClient k8sJobClient = localMockK8sJobClient.select("any"); + K8sResourceContext k8sResourceContext = + new K8sResourceContext(new PodConfig(), "local", "local", "image", "local", null); + K8sPodResource k8sResource = k8sJobClient.create(k8sResourceContext); + Assert.assertNotNull(k8sResource); + } +} From c857c56d4af288ea0ee00111184f114a3e0e4cb5 Mon Sep 17 00:00:00 2001 From: IL MARE Date: Thu, 24 Oct 2024 10:57:37 +0800 Subject: [PATCH 010/118] fix(global-search): failed to sync database which get 'PENDING' status and 'object_last_sync_time' is null (#3731) * fix failed to sync database * update submodule --- .../odc/metadb/connection/DatabaseRepositoryTest.java | 9 +++++---- .../odc/metadb/connection/DatabaseRepository.java | 7 ++++--- .../service/connection/database/DatabaseService.java | 5 +++-- .../odc/service/connection/table/TableService.java | 8 +++----- .../odc/service/db/schema/GlobalSearchProperties.java | 10 +++++----- .../schema/syncer/column/AbstractDBColumnSyncer.java | 4 ++++ .../schema/syncer/object/AbstractDBObjectSyncer.java | 4 ++++ 7 files changed, 28 insertions(+), 19 deletions(-) diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java index 2d34beb4da..3b3e74ac21 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java @@ -57,9 +57,10 @@ public void setObjectSyncStatusByObjectLastSyncTimeBefore_noObjectMatched_setNot d2.setObjectLastSyncTime(syncTime); d2.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); this.databaseRepository.saveAll(Arrays.asList(d1, d2)); - int affectRows = this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore( - DBObjectSyncStatus.SYNCING, DBObjectSyncStatus.SYNCING, - new Date(System.currentTimeMillis() - 86400 * 2)); + int affectRows = + this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeIsNullOrBefore( + DBObjectSyncStatus.SYNCING, DBObjectSyncStatus.SYNCING, + new Date(System.currentTimeMillis() - 86400 * 2)); Assert.assertEquals(0, affectRows); } @@ -72,7 +73,7 @@ public void setObjectSyncStatusByObjectLastSyncTimeBefore_oneObjectMatched_setSu d2.setObjectLastSyncTime(new Date(System.currentTimeMillis() - 86400)); d2.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); this.databaseRepository.saveAll(Arrays.asList(d1, d2)); - this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore( + this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeIsNullOrBefore( DBObjectSyncStatus.INITIALIZED, DBObjectSyncStatus.SYNCED, new Date(System.currentTimeMillis() - 86400 / 2)); Set actual = this.databaseRepository.findAll().stream() diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java index 094953a252..1e1a27f678 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java @@ -74,9 +74,10 @@ public interface DatabaseRepository extends JpaRepository, @Modifying @Transactional @Query(value = "update connect_database t set t.object_sync_status = :#{#status.name()} " - + "where t.object_sync_status = :#{#originalStatus.name()} and t.object_last_sync_time < :syncTime", - nativeQuery = true) - int setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore(@Param("status") DBObjectSyncStatus status, + + "where t.object_sync_status = :#{#originalStatus.name()} " + + "and (t.object_last_sync_time < :syncTime or t.object_last_sync_time is null)", nativeQuery = true) + int setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeIsNullOrBefore( + @Param("status") DBObjectSyncStatus status, @Param("originalStatus") DBObjectSyncStatus originalStatus, @Param("syncTime") Date syncTime); @Modifying diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index e36d90c39a..8404c00fc7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -831,8 +831,9 @@ public void updateObjectLastSyncTimeAndStatus(@NotNull Long databaseId, @Transactional(rollbackFor = Exception.class) public void refreshExpiredPendingDBObjectStatus() { Date syncDate = new Date(System.currentTimeMillis() - this.globalSearchProperties.getMaxPendingMillis()); - int affectRows = this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore( - DBObjectSyncStatus.INITIALIZED, DBObjectSyncStatus.PENDING, syncDate); + int affectRows = + this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeIsNullOrBefore( + DBObjectSyncStatus.INITIALIZED, DBObjectSyncStatus.PENDING, syncDate); log.info("Refresh outdated pending objects status, syncDate={}, affectRows={}", syncDate, affectRows); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java index f32066dca7..3aba1649d8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java @@ -134,9 +134,8 @@ public List
list(@NonNull @Valid QueryTableParams params) throws SQLExcep } private void generateListAndSyncDBTablesByTableType(QueryTableParams params, Database database, - ConnectionConfig dataSource, - List
tables, - Connection conn, DBObjectType tableType, TableExtensionPoint tableExtension) throws InterruptedException { + ConnectionConfig dataSource, List
tables, Connection conn, DBObjectType tableType, + TableExtensionPoint tableExtension) throws InterruptedException { Set latestTableNames = tableExtension.list(conn, database.getName(), tableType) .stream().map(DBObjectIdentity::getName).collect(Collectors.toCollection(LinkedHashSet::new)); if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { @@ -182,8 +181,7 @@ public void syncDBTables(@NotNull Connection connection, @NotNull Database datab } private void syncDBTables(@NotNull Connection connection, @NotNull Database database, - @NotNull DialectType dialectType, - @NotNull DBSchemaSyncer syncer) throws InterruptedException { + @NotNull DialectType dialectType, @NotNull DBSchemaSyncer syncer) throws InterruptedException { Lock lock = lockRegistry .obtain(dbSchemaSyncService.getSyncDBObjectLockKey(database.getDataSource().getId(), database.getId())); if (!lock.tryLock(3, TimeUnit.SECONDS)) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java index 1024d4df08..8d581501b5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java @@ -40,12 +40,12 @@ public class GlobalSearchProperties { @Value("${odc.database.schema.global-search.max-pending-hours:1}") private long maxPendingHours; + public long getMaxPendingHours() { + return this.maxPendingHours <= 0 ? 1 : this.maxPendingHours; + } + public long getMaxPendingMillis() { - long maxPendingHours = this.maxPendingHours; - if (this.maxPendingHours <= 0) { - maxPendingHours = 1; - } - return TimeUnit.MILLISECONDS.convert(this.maxPendingHours, TimeUnit.HOURS); + return TimeUnit.MILLISECONDS.convert(getMaxPendingHours(), TimeUnit.HOURS); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java index 0d65905f4a..28962964a9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java @@ -36,6 +36,7 @@ import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; import com.oceanbase.odc.service.db.schema.syncer.DBSchemaSyncer; import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.tools.dbbrowser.model.DBObjectType; @@ -58,6 +59,9 @@ public abstract class AbstractDBColumnSyncer implement @Override public void sync(@NonNull Connection connection, @NonNull Database database, @NonNull DialectType dialectType) { + if (database.getType() != DatabaseType.PHYSICAL) { + return; + } T extensionPoint = getExtensionPoint(dialectType); if (extensionPoint == null) { return; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java index 1007fc328b..a84bba2702 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java @@ -30,6 +30,7 @@ import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; import com.oceanbase.odc.service.db.schema.syncer.DBSchemaSyncer; import com.oceanbase.odc.service.plugin.SchemaPluginUtil; @@ -51,6 +52,9 @@ public abstract class AbstractDBObjectSyncer implement @Override public void sync(@NonNull Connection connection, @NonNull Database database, @NonNull DialectType dialectType) { + if (database.getType() != DatabaseType.PHYSICAL) { + return; + } T extensionPoint = getExtensionPoint(dialectType); if (extensionPoint == null) { return; From dec03d108631ee360f8d908873b69fcbda87693b Mon Sep 17 00:00:00 2001 From: IL MARE Date: Fri, 25 Oct 2024 15:45:00 +0800 Subject: [PATCH 011/118] build: upgrade odc version from 4.3.2 to 4.3.3 #3732 --- .gitmodules | 2 +- client | 2 +- distribution/odc-server-VER.txt | 2 +- pom.xml | 2 +- server/3rd-party/Libinjection/pom.xml | 2 +- server/integration-test/pom.xml | 2 +- server/modules/pom.xml | 2 +- server/modules/sample-module/pom.xml | 2 +- server/odc-common/pom.xml | 2 +- server/odc-core/pom.xml | 2 +- server/odc-migrate/pom.xml | 2 +- server/odc-server/pom.xml | 2 +- server/odc-service/pom.xml | 2 +- server/odc-test/pom.xml | 2 +- server/plugins/connect-plugin-api/pom.xml | 2 +- server/plugins/connect-plugin-doris/pom.xml | 2 +- server/plugins/connect-plugin-mysql/pom.xml | 2 +- server/plugins/connect-plugin-ob-mysql/pom.xml | 2 +- server/plugins/connect-plugin-ob-oracle/pom.xml | 2 +- server/plugins/connect-plugin-oracle/pom.xml | 2 +- server/plugins/connect-plugin-postgres/pom.xml | 2 +- server/plugins/pom.xml | 2 +- server/plugins/sample-plugin-api/pom.xml | 2 +- server/plugins/sample-plugin/pom.xml | 2 +- server/plugins/schema-plugin-api/pom.xml | 2 +- server/plugins/schema-plugin-doris/pom.xml | 2 +- server/plugins/schema-plugin-mysql/pom.xml | 2 +- server/plugins/schema-plugin-ob-mysql/pom.xml | 2 +- server/plugins/schema-plugin-ob-oracle/pom.xml | 2 +- server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml | 2 +- server/plugins/schema-plugin-oracle/pom.xml | 2 +- server/plugins/schema-plugin-postgres/pom.xml | 2 +- server/plugins/task-plugin-api/pom.xml | 2 +- server/plugins/task-plugin-doris/pom.xml | 2 +- server/plugins/task-plugin-mysql/pom.xml | 2 +- server/plugins/task-plugin-ob-mysql/pom.xml | 2 +- server/plugins/task-plugin-ob-oracle/pom.xml | 2 +- server/plugins/task-plugin-oracle/pom.xml | 2 +- server/starters/desktop-starter/pom.xml | 2 +- server/starters/pom.xml | 2 +- server/starters/web-starter/pom.xml | 2 +- 41 files changed, 41 insertions(+), 41 deletions(-) diff --git a/.gitmodules b/.gitmodules index bb0072910f..f52bde7184 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "client"] path = client url = https://github.com/oceanbase/odc-client.git - branch = dev-4.3.2 + branch = dev-4.3.3 [submodule "build-resource"] path = build-resource url = https://github.com/oceanbase/odc-build-resource.git diff --git a/client b/client index 500782eeb6..0926668940 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 500782eeb65b5fa2bbb02c2595a499c2b07c49e7 +Subproject commit 0926668940c112b0c5a4c21a16280823a7ef567f diff --git a/distribution/odc-server-VER.txt b/distribution/odc-server-VER.txt index cc2fbe89b6..e91d9be2a8 100644 --- a/distribution/odc-server-VER.txt +++ b/distribution/odc-server-VER.txt @@ -1 +1 @@ -4.3.2 +4.3.3 diff --git a/pom.xml b/pom.xml index 07c1c566b4..847939893e 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ pom OceanBase Developer Center https://github.com/oceanbase/odc - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT server/3rd-party/Libinjection server/odc-test diff --git a/server/3rd-party/Libinjection/pom.xml b/server/3rd-party/Libinjection/pom.xml index ab117e953e..96db6dc34d 100644 --- a/server/3rd-party/Libinjection/pom.xml +++ b/server/3rd-party/Libinjection/pom.xml @@ -8,7 +8,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../../pom.xml Libinjection diff --git a/server/integration-test/pom.xml b/server/integration-test/pom.xml index 0c9d27fcfe..a24391df3a 100644 --- a/server/integration-test/pom.xml +++ b/server/integration-test/pom.xml @@ -7,7 +7,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml integration-test diff --git a/server/modules/pom.xml b/server/modules/pom.xml index 8cb1f293c1..d4efe7d7c4 100644 --- a/server/modules/pom.xml +++ b/server/modules/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml pom diff --git a/server/modules/sample-module/pom.xml b/server/modules/sample-module/pom.xml index ac0d00fb4c..d3d1ceb16c 100644 --- a/server/modules/sample-module/pom.xml +++ b/server/modules/sample-module/pom.xml @@ -5,7 +5,7 @@ module-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml sample-module diff --git a/server/odc-common/pom.xml b/server/odc-common/pom.xml index 4ddc210be7..4e4a9cf2fc 100644 --- a/server/odc-common/pom.xml +++ b/server/odc-common/pom.xml @@ -7,7 +7,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-common diff --git a/server/odc-core/pom.xml b/server/odc-core/pom.xml index 85ab44fab0..36a3bb0e07 100644 --- a/server/odc-core/pom.xml +++ b/server/odc-core/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-core diff --git a/server/odc-migrate/pom.xml b/server/odc-migrate/pom.xml index 34470f27de..9b79fcade3 100644 --- a/server/odc-migrate/pom.xml +++ b/server/odc-migrate/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-migrate diff --git a/server/odc-server/pom.xml b/server/odc-server/pom.xml index 053792715b..3eec2dae2e 100644 --- a/server/odc-server/pom.xml +++ b/server/odc-server/pom.xml @@ -6,7 +6,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-server diff --git a/server/odc-service/pom.xml b/server/odc-service/pom.xml index fb715cdd70..3cde90b1d5 100644 --- a/server/odc-service/pom.xml +++ b/server/odc-service/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-service diff --git a/server/odc-test/pom.xml b/server/odc-test/pom.xml index d3bf3ebbd6..b11dd83467 100644 --- a/server/odc-test/pom.xml +++ b/server/odc-test/pom.xml @@ -7,7 +7,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-test diff --git a/server/plugins/connect-plugin-api/pom.xml b/server/plugins/connect-plugin-api/pom.xml index f26d4921e8..5623a7dc1a 100644 --- a/server/plugins/connect-plugin-api/pom.xml +++ b/server/plugins/connect-plugin-api/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/connect-plugin-doris/pom.xml b/server/plugins/connect-plugin-doris/pom.xml index 96581d2f12..7fa1cb09b2 100644 --- a/server/plugins/connect-plugin-doris/pom.xml +++ b/server/plugins/connect-plugin-doris/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/connect-plugin-mysql/pom.xml b/server/plugins/connect-plugin-mysql/pom.xml index a10ac956f1..626a25c9d6 100644 --- a/server/plugins/connect-plugin-mysql/pom.xml +++ b/server/plugins/connect-plugin-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/connect-plugin-ob-mysql/pom.xml b/server/plugins/connect-plugin-ob-mysql/pom.xml index 0645701904..156a6b6d33 100644 --- a/server/plugins/connect-plugin-ob-mysql/pom.xml +++ b/server/plugins/connect-plugin-ob-mysql/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/connect-plugin-ob-oracle/pom.xml b/server/plugins/connect-plugin-ob-oracle/pom.xml index c642b58922..d1b6a0b10e 100644 --- a/server/plugins/connect-plugin-ob-oracle/pom.xml +++ b/server/plugins/connect-plugin-ob-oracle/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/connect-plugin-oracle/pom.xml b/server/plugins/connect-plugin-oracle/pom.xml index f7ce69d8be..ac3f6294a2 100644 --- a/server/plugins/connect-plugin-oracle/pom.xml +++ b/server/plugins/connect-plugin-oracle/pom.xml @@ -21,7 +21,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/connect-plugin-postgres/pom.xml b/server/plugins/connect-plugin-postgres/pom.xml index 4d3b4ea999..bd55295235 100644 --- a/server/plugins/connect-plugin-postgres/pom.xml +++ b/server/plugins/connect-plugin-postgres/pom.xml @@ -22,7 +22,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml connect-plugin-postgres diff --git a/server/plugins/pom.xml b/server/plugins/pom.xml index 053993f888..21e661d754 100644 --- a/server/plugins/pom.xml +++ b/server/plugins/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml pom diff --git a/server/plugins/sample-plugin-api/pom.xml b/server/plugins/sample-plugin-api/pom.xml index 5a6f36ccec..e5c7add671 100644 --- a/server/plugins/sample-plugin-api/pom.xml +++ b/server/plugins/sample-plugin-api/pom.xml @@ -5,7 +5,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml sample-plugin-api diff --git a/server/plugins/sample-plugin/pom.xml b/server/plugins/sample-plugin/pom.xml index 1c85f05790..919e72c4af 100644 --- a/server/plugins/sample-plugin/pom.xml +++ b/server/plugins/sample-plugin/pom.xml @@ -5,7 +5,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml sample-plugin diff --git a/server/plugins/schema-plugin-api/pom.xml b/server/plugins/schema-plugin-api/pom.xml index 77b3c9a764..4847c90823 100644 --- a/server/plugins/schema-plugin-api/pom.xml +++ b/server/plugins/schema-plugin-api/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-doris/pom.xml b/server/plugins/schema-plugin-doris/pom.xml index dad1ff7783..4cb0fe5e2c 100644 --- a/server/plugins/schema-plugin-doris/pom.xml +++ b/server/plugins/schema-plugin-doris/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-mysql/pom.xml b/server/plugins/schema-plugin-mysql/pom.xml index 463d216374..e605736f68 100644 --- a/server/plugins/schema-plugin-mysql/pom.xml +++ b/server/plugins/schema-plugin-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-ob-mysql/pom.xml b/server/plugins/schema-plugin-ob-mysql/pom.xml index 1655445e1b..246fc7d92b 100644 --- a/server/plugins/schema-plugin-ob-mysql/pom.xml +++ b/server/plugins/schema-plugin-ob-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml schema-plugin-ob-mysql diff --git a/server/plugins/schema-plugin-ob-oracle/pom.xml b/server/plugins/schema-plugin-ob-oracle/pom.xml index b35cbbec2f..a6e89f54ad 100644 --- a/server/plugins/schema-plugin-ob-oracle/pom.xml +++ b/server/plugins/schema-plugin-ob-oracle/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml b/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml index 49a1427d78..4d0456baa2 100644 --- a/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml +++ b/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-oracle/pom.xml b/server/plugins/schema-plugin-oracle/pom.xml index bd234f03c3..1085881197 100644 --- a/server/plugins/schema-plugin-oracle/pom.xml +++ b/server/plugins/schema-plugin-oracle/pom.xml @@ -23,7 +23,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-postgres/pom.xml b/server/plugins/schema-plugin-postgres/pom.xml index c91c5440b3..cd517d9b6c 100644 --- a/server/plugins/schema-plugin-postgres/pom.xml +++ b/server/plugins/schema-plugin-postgres/pom.xml @@ -22,7 +22,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/task-plugin-api/pom.xml b/server/plugins/task-plugin-api/pom.xml index c417da8a22..2eed9f3b42 100644 --- a/server/plugins/task-plugin-api/pom.xml +++ b/server/plugins/task-plugin-api/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/task-plugin-doris/pom.xml b/server/plugins/task-plugin-doris/pom.xml index f12956bed7..2f127b508c 100644 --- a/server/plugins/task-plugin-doris/pom.xml +++ b/server/plugins/task-plugin-doris/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-mysql/pom.xml b/server/plugins/task-plugin-mysql/pom.xml index e553272924..1441170c0d 100644 --- a/server/plugins/task-plugin-mysql/pom.xml +++ b/server/plugins/task-plugin-mysql/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-ob-mysql/pom.xml b/server/plugins/task-plugin-ob-mysql/pom.xml index df29e3d63d..916cacdd0e 100644 --- a/server/plugins/task-plugin-ob-mysql/pom.xml +++ b/server/plugins/task-plugin-ob-mysql/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-ob-oracle/pom.xml b/server/plugins/task-plugin-ob-oracle/pom.xml index 838967f209..29ff2da494 100644 --- a/server/plugins/task-plugin-ob-oracle/pom.xml +++ b/server/plugins/task-plugin-ob-oracle/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-oracle/pom.xml b/server/plugins/task-plugin-oracle/pom.xml index 923fd2a552..cc9c9124e1 100644 --- a/server/plugins/task-plugin-oracle/pom.xml +++ b/server/plugins/task-plugin-oracle/pom.xml @@ -21,7 +21,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT 4.0.0 diff --git a/server/starters/desktop-starter/pom.xml b/server/starters/desktop-starter/pom.xml index d21ff1931c..9dcdd92000 100644 --- a/server/starters/desktop-starter/pom.xml +++ b/server/starters/desktop-starter/pom.xml @@ -5,7 +5,7 @@ starter-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml desktop-starter diff --git a/server/starters/pom.xml b/server/starters/pom.xml index ce350454ec..ed3dcbb7b0 100644 --- a/server/starters/pom.xml +++ b/server/starters/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml pom diff --git a/server/starters/web-starter/pom.xml b/server/starters/web-starter/pom.xml index 75c369f7ea..a0300c5e11 100644 --- a/server/starters/web-starter/pom.xml +++ b/server/starters/web-starter/pom.xml @@ -5,7 +5,7 @@ starter-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml web-starter From a9077917ff06d6a33b467f2502cbf65be8004fa5 Mon Sep 17 00:00:00 2001 From: "yh263208@oceanbase.com" Date: Tue, 29 Oct 2024 19:46:23 +0800 Subject: [PATCH 012/118] update submodule --- client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client b/client index 59e4b8ad86..39844a21e4 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 59e4b8ad86c21916692d4bc3c4db271d09ea2e70 +Subproject commit 39844a21e4d76aa1f70ea67c981c74ac56ba38fb From 9612ff91ff89d6d9ba9d95eb142ea065ac6b264c Mon Sep 17 00:00:00 2001 From: "yh263208@oceanbase.com" Date: Wed, 30 Oct 2024 10:38:55 +0800 Subject: [PATCH 013/118] resp to comments --- .../task/schedule/daemon/DoCancelingJob.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java index 8d04def79e..e16df67e8e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java @@ -27,6 +27,7 @@ import com.oceanbase.odc.service.task.config.JobConfigurationValidator; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; import com.oceanbase.odc.service.task.enums.JobStatus; +import com.oceanbase.odc.service.task.enums.TaskRunMode; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; import com.oceanbase.odc.service.task.listener.JobTerminateEvent; import com.oceanbase.odc.service.task.schedule.JobIdentity; @@ -73,14 +74,11 @@ private void cancelJob(TaskFrameworkService taskFrameworkService, JobEntity jobE lockedEntity.getId(), lockedEntity.getStatus()); return; } - JobStatus currentStatus = taskFrameworkService.find(lockedEntity.getId()).getStatus(); - if (currentStatus.isTerminated()) { - // the job terminated before we update it to CANCELED - log.info("Job is already terminated, jobId={},currentStatus={}", lockedEntity.getId(), - currentStatus); - getConfiguration().getEventPublisher().publishEvent( - new JobTerminateEvent(JobIdentity.of(lockedEntity.getId()), currentStatus)); - return; + // mark resource as released + if (TaskRunMode.K8S == lockedEntity.getRunMode()) { + ResourceManagerUtil.markResourceReleased(lockedEntity, lockedEntity.getExecutorIdentifier(), + getConfiguration().getResourceManager()); + log.info("DoCancelingJob release resource for job = {}", jobEntity); } // For transaction atomic, first update to CANCELED, then stop remote job in executor, // if stop remote failed, transaction will be rollback From 94ca890cec9168b34b8f6088911a9dbeab463ade Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Wed, 30 Oct 2024 18:05:06 +0800 Subject: [PATCH 014/118] feat(sqlcheck): add sql affect rows support for oracle and ob oracle (#3735) * feat: add support for OBOracleAffectedRowsExceedLimit * Solve the unit test error * support Oracle * modify according to commend * modify according to commend * modify according to commend * modify according to commend --- .../init/regulation-rule-applying.yaml | 8 ++ .../init/regulation-rule-metadata.yaml | 4 + .../factory/SqlAffectedRowsFactory.java | 20 ++- .../factory/TruncateTableExistsFactory.java | 2 +- .../Unable2JudgeAffectedRowsFactory.java | 4 +- .../sqlcheck/model/SqlCheckRuleType.java | 2 +- .../rule/BaseAffectedRowsExceedLimit.java | 70 ++++++++++ .../rule/MySQLAffectedRowsExceedLimit.java | 105 ++------------ .../rule/OracleAffectedRowsExceedLimit.java | 128 ++++++++++++++++++ .../sqlcheck/rule/TruncateTableExists.java | 2 +- .../sqlcheck/OracleSqlCheckerTest.java | 2 +- 11 files changed, 247 insertions(+), 100 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleAffectedRowsExceedLimit.java diff --git a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml index 26050692c1..a949db4027 100644 --- a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml +++ b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml @@ -449,6 +449,8 @@ appliedDialectTypes: - 'OB_MYSQL' - 'MYSQL' + - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count}":1000}' - enabled: 1 level: 0 @@ -950,6 +952,8 @@ appliedDialectTypes: - 'OB_MYSQL' - 'MYSQL' + - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count}":10000}' - enabled: 1 level: 0 @@ -1451,6 +1455,8 @@ appliedDialectTypes: - 'OB_MYSQL' - 'MYSQL' + - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count}":1000}' - enabled: 1 level: 1 @@ -1942,6 +1948,8 @@ appliedDialectTypes: - 'OB_MYSQL' - 'MYSQL' + - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count}":1000}' - enabled: 1 level: 1 diff --git a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml index 0a84e57a55..d519181ab1 100644 --- a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml +++ b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml @@ -1416,6 +1416,10 @@ value: OB_MYSQL - label: SUPPORTED_DIALECT_TYPE value: MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: OB_ORACLE + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count.description} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/SqlAffectedRowsFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/SqlAffectedRowsFactory.java index f11a590c11..abf5b4d2ad 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/SqlAffectedRowsFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/SqlAffectedRowsFactory.java @@ -24,10 +24,12 @@ import com.oceanbase.odc.service.sqlcheck.SqlCheckRuleFactory; import com.oceanbase.odc.service.sqlcheck.model.SqlCheckRuleType; import com.oceanbase.odc.service.sqlcheck.rule.MySQLAffectedRowsExceedLimit; +import com.oceanbase.odc.service.sqlcheck.rule.OracleAffectedRowsExceedLimit; import lombok.NonNull; public class SqlAffectedRowsFactory implements SqlCheckRuleFactory { + public static final long DEFAULT_MAX_SQL_AFFECTED_ROWS = 1000L; private final JdbcOperations jdbc; @@ -43,9 +45,21 @@ public SqlCheckRuleType getSupportsType() { @Override public SqlCheckRule generate(@NonNull DialectType dialectType, Map parameters) { String key = getParameterNameKey("allowed-max-sql-affected-count"); - if (parameters == null || parameters.isEmpty() || parameters.get(key) == null) { - return new MySQLAffectedRowsExceedLimit(1000L, dialectType, jdbc); + long maxSqlAffectedRows = DEFAULT_MAX_SQL_AFFECTED_ROWS; + if (parameters != null && !parameters.isEmpty() && parameters.get(key) != null) { + maxSqlAffectedRows = Long.valueOf(parameters.get(key).toString()); + } + switch (dialectType) { + case ORACLE: + case OB_ORACLE: + return new OracleAffectedRowsExceedLimit(maxSqlAffectedRows, dialectType, + jdbc); + case MYSQL: + case OB_MYSQL: + return new MySQLAffectedRowsExceedLimit(maxSqlAffectedRows, dialectType, + jdbc); + default: + throw new IllegalArgumentException("Unsupported dialect type: " + dialectType); } - return new MySQLAffectedRowsExceedLimit(Long.valueOf(parameters.get(key).toString()), dialectType, jdbc); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/TruncateTableExistsFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/TruncateTableExistsFactory.java index 9c410f575a..1caadad998 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/TruncateTableExistsFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/TruncateTableExistsFactory.java @@ -29,7 +29,7 @@ public class TruncateTableExistsFactory implements SqlCheckRuleFactory { @Override public SqlCheckRuleType getSupportsType() { - return SqlCheckRuleType.TRUNCATE_TBLE_EXISTS; + return SqlCheckRuleType.TRUNCATE_TABLE_EXISTS; } @Override diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/Unable2JudgeAffectedRowsFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/Unable2JudgeAffectedRowsFactory.java index 224931cbe2..434cea6246 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/Unable2JudgeAffectedRowsFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/Unable2JudgeAffectedRowsFactory.java @@ -23,7 +23,7 @@ import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; import com.oceanbase.odc.service.sqlcheck.SqlCheckRuleFactory; import com.oceanbase.odc.service.sqlcheck.model.SqlCheckRuleType; -import com.oceanbase.odc.service.sqlcheck.rule.MySQLAffectedRowsExceedLimit; +import com.oceanbase.odc.service.sqlcheck.rule.BaseAffectedRowsExceedLimit; import com.oceanbase.odc.service.sqlcheck.rule.Unable2JudgeAffectedRows; import lombok.NonNull; @@ -44,7 +44,7 @@ public SqlCheckRuleType getSupportsType() { @Override public SqlCheckRule generate(@NonNull DialectType dialectType, Map parameters) { SqlAffectedRowsFactory sqlAffectedRowsFactory = new SqlAffectedRowsFactory(this.jdbc); - MySQLAffectedRowsExceedLimit targetRule = (MySQLAffectedRowsExceedLimit) sqlAffectedRowsFactory + BaseAffectedRowsExceedLimit targetRule = (BaseAffectedRowsExceedLimit) sqlAffectedRowsFactory .generate(dialectType, parameters); return new Unable2JudgeAffectedRows(targetRule); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java index 9001b43a42..752558df8f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java @@ -243,7 +243,7 @@ public enum SqlCheckRuleType implements Translatable { /** * Truncate table 语句存在 */ - TRUNCATE_TBLE_EXISTS("truncate-table-exists"), + TRUNCATE_TABLE_EXISTS("truncate-table-exists"), /** * Restrict the number of lines affected by SQL */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java index 5e7e204f68..7c2953e1d2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java @@ -18,6 +18,8 @@ import java.util.Collections; import java.util.List; +import org.springframework.jdbc.core.JdbcOperations; + import com.oceanbase.odc.service.sqlcheck.SqlCheckContext; import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; import com.oceanbase.odc.service.sqlcheck.SqlCheckUtil; @@ -63,6 +65,74 @@ public List check(@NonNull Statement statement, @NonNull SqlChec return Collections.emptyList(); } + /** + * OB execute 'explain' statement + * + * @param originalSql target sql + * @param jdbcOperations jdbc Object + * @return affected rows + */ + public long getOBAffectedRows(String originalSql, JdbcOperations jdbcOperations) { + /** + * 
+         *     obclient> explain delete from T1 where 1=1;
+         * +-------------------------------------------------------+
+         * | Query Plan                                            |
+         * +-------------------------------------------------------+
+         * | =================================================     |
+         * | |ID|OPERATOR         |NAME|EST.ROWS|EST.TIME(us)|     |
+         * | -------------------------------------------------     |
+         * | |0 |DELETE           |    |2       |21          |     |
+         * | |1 |└─TABLE FULL SCAN|t1  |2       |5           |     |
+         * | =================================================     |
+         * | Outputs & filters:                                    |
+         * | -------------------------------------                 |
+         * |   0 - output(nil), filter(nil)                        |
+         * |       table_columns([{t1: ({t1: (t1.id)})}])          |
+         * |   1 - output([t1.id]), filter(nil), rowset=16         |
+         * |       access([t1.id]), partitions(p0)                 |
+         * |       is_index_back=false, is_global_index=false,     |
+         * |       range_key([t1.id]), range(MIN ; MAX)always true |
+         * +-------------------------------------------------------+
+         * 14 rows in set (0.00 sec)
+         *
+ */ + String explainSql = "EXPLAIN " + originalSql; + List queryResults = jdbcOperations.query(explainSql, (rs, rowNum) -> rs.getString("Query Plan")); + long estRowsValue = 0; + for (int rowNum = 3; rowNum < queryResults.size(); rowNum++) { + String resultRow = queryResults.get(rowNum); + estRowsValue = getEstRowsValue(resultRow); + if (estRowsValue != 0) { + break; + } + } + return estRowsValue; + } + + public long getEstRowsValue(String singleRow) { + String[] parts = singleRow.split("\\|"); + if (parts.length > 5) { + String value = parts[4].trim(); + return parseLong(value); + } + return 0; + } + + /** + * Safely parse a long value. + * + * @param value string to parse + * @return parsed long or 0 if parsing fails + */ + private long parseLong(String value) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + return 0; + } + } + public abstract long getStatementAffectedRows(Statement statement) throws Exception; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLAffectedRowsExceedLimit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLAffectedRowsExceedLimit.java index 4b8b869f6c..2b5b4254a9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLAffectedRowsExceedLimit.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLAffectedRowsExceedLimit.java @@ -16,11 +16,9 @@ package com.oceanbase.odc.service.sqlcheck.rule; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.collections4.CollectionUtils; import org.springframework.jdbc.core.JdbcOperations; @@ -74,28 +72,20 @@ public long getStatementAffectedRows(Statement statement) { long affectedRows = 0; if (statement instanceof Update || statement instanceof Delete || statement instanceof Insert) { String explainSql = "EXPLAIN " + statement.getText(); - try { - if (this.jdbcOperations == null) { - log.warn("JdbcOperations is null, please check your connection"); - return -1; - } else { - switch (this.dialectType) { - case MYSQL: - affectedRows = (statement instanceof Insert) - ? getMySqlAffectedRowsByCount((Insert) statement) - : getMySqlAffectedRowsByExplain(explainSql, this.jdbcOperations); - break; - case OB_MYSQL: - affectedRows = getOBMySqlAffectedRows(explainSql, this.jdbcOperations); - break; - default: - log.warn("Unsupported dialect type: {}", this.dialectType); - break; - } - } - } catch (Exception e) { - log.warn("Error in calling getAffectedRows method", e); - affectedRows = -1; + if (this.jdbcOperations == null) { + throw new IllegalStateException("JdbcOperations is null, please check your connection"); + } + switch (this.dialectType) { + case MYSQL: + affectedRows = (statement instanceof Insert) + ? getMySqlAffectedRowsByCount((Insert) statement) + : getMySqlAffectedRowsByExplain(explainSql, this.jdbcOperations); + break; + case OB_MYSQL: + affectedRows = getOBAffectedRows(statement.getText(), this.jdbcOperations); + break; + default: + throw new UnsupportedOperationException("Unsupported dialect type: " + this.dialectType); } } return affectedRows; @@ -163,71 +153,4 @@ private long getMySqlAffectedRowsByExplain(String explainSql, JdbcOperations jdb } } - /** - * OBMySQL execute 'explain' statement - * - * @param explainSql target sql - * @param jdbc jdbc Object - * @return affected rows - */ - private long getOBMySqlAffectedRows(String explainSql, JdbcOperations jdbc) { - /** - * 
-         *
-         *     explain result (json):
-         *
-         *     ==================================================    --rowNum = 1
-         *     |ID|OPERATOR          |NAME|EST.ROWS|EST.TIME(us)|    --rowNum = 2
-         *     0 |DISTRIBUTED UPDATE|    |1       |37          |     --rowNum = 3
-         *     1 |└─TABLE GET       |user|1       |5           |     --rowNum = 4
-         *     ==================================================    --rowNum = 5
-         *     ...
-         *
-         *
- */ - try { - AtomicBoolean ifFindAffectedRow = new AtomicBoolean(false); - List queryResults = jdbc.query(explainSql, (rs, rowNum) -> rs.getString("Query Plan")); - List resultSet = new ArrayList<>(); - for (int rowNum = 0; rowNum < queryResults.size(); rowNum++) { - String resultRow = queryResults.get(rowNum); - if (!ifFindAffectedRow.get() && rowNum > 2) { - // Find the first non-null value in the column 'EST.ROWS' - long estRowsValue = getEstRowsValue(resultRow); - if (estRowsValue != 0) { - ifFindAffectedRow.set(true); - resultSet.add(estRowsValue); - } - } - resultSet.add(null); - } - - Long firstNonNullResult = resultSet.stream() - .filter(Objects::nonNull) - .findFirst() - .orElse(null); - - return firstNonNullResult != null ? firstNonNullResult : 0; - - } catch (Exception e) { - log.warn("OBMySQL mode: Error in execute " + explainSql + " failed. ", e); - return -1; - } - } - - /** - * parse explain result set - * - * @param singleRow row - * @return affected rows - */ - private long getEstRowsValue(String singleRow) { - String[] parts = singleRow.split("\\|"); - if (parts.length > 4) { - String value = parts[4].trim(); - return Long.parseLong(value); - } - return 0; - } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleAffectedRowsExceedLimit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleAffectedRowsExceedLimit.java new file mode 100644 index 0000000000..86abf9cc0a --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleAffectedRowsExceedLimit.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.sqlcheck.rule; + +import java.util.Arrays; +import java.util.List; + +import org.springframework.jdbc.core.JdbcOperations; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.tools.sqlparser.statement.Statement; +import com.oceanbase.tools.sqlparser.statement.delete.Delete; +import com.oceanbase.tools.sqlparser.statement.insert.Insert; +import com.oceanbase.tools.sqlparser.statement.update.Update; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +/** + * @description:{@link OracleAffectedRowsExceedLimit} + * + * @author: zijia.cj + * @date: 2024/10/24 13:06 + * @since: 4.3.3 + */ +@Slf4j +public class OracleAffectedRowsExceedLimit extends BaseAffectedRowsExceedLimit { + public static final String ODC_TEMP_EXPLAIN_STATEMENT_ID = "ODC_TEMP_EXPLAIN_STATEMENT_ID"; + + private final JdbcOperations jdbcOperations; + private final DialectType dialectType; + + public OracleAffectedRowsExceedLimit(@NonNull Long maxSqlAffectedRows, DialectType dialectType, + JdbcOperations jdbcOperations) { + super(maxSqlAffectedRows); + this.jdbcOperations = jdbcOperations; + this.dialectType = dialectType; + } + + /** + * Get supported database types + */ + @Override + public List getSupportsDialectTypes() { + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); + } + + /** + * Base method implemented by Oracle types + */ + @Override + public long getStatementAffectedRows(Statement statement) { + long affectedRows = 0; + if (statement instanceof Update || statement instanceof Delete || statement instanceof Insert) { + String originalSql = statement.getText(); + if (this.jdbcOperations == null) { + throw new IllegalStateException("JdbcOperations is null, please check your connection"); + } + switch (this.dialectType) { + case ORACLE: + affectedRows = getOracleAffectedRows(originalSql, this.jdbcOperations); + break; + case OB_ORACLE: + affectedRows = getOBAffectedRows(originalSql, this.jdbcOperations); + break; + default: + throw new UnsupportedOperationException("Unsupported dialect type: " + this.dialectType); + } + } + return affectedRows; + } + + private long getOracleAffectedRows(String originalSql, JdbcOperations jdbcOperations) { + /** + * 
+         *     Plan hash value: 775918519
+         *
+         * -------------------------------------------------------------------
+         * | Id  | Operation          | Name | Rows  | Cost (%CPU)| Time     |
+         * -------------------------------------------------------------------
+         * |   0 | DELETE STATEMENT   |      |    11 |     3   (0)| 00:00:01 |
+         * |   1 |  DELETE            | T1   |       |            |          |
+         * |   2 |   TABLE ACCESS FULL| T1   |    11 |     3   (0)| 00:00:01 |
+         * -------------------------------------------------------------------
+         *
+         * Query Block Name / Object Alias (identified by operation id):
+         * -------------------------------------------------------------
+         *
+         *    1 - DEL$1
+         *    2 - DEL$1 / T1@DEL$1
+         *
+         * Column Projection Information (identified by operation id):
+         * -----------------------------------------------------------
+         *
+         *    2 - "T1".ROWID[ROWID,10]
+         *
+ */ + String SetPlanSql = + "EXPLAIN PLAN SET STATEMENT_ID = '" + ODC_TEMP_EXPLAIN_STATEMENT_ID + "' FOR " + originalSql; + jdbcOperations.execute(SetPlanSql); + String getPlanSql = + "SELECT * FROM TABLE(DBMS_XPLAN.DISPLAY('PLAN_TABLE', '" + ODC_TEMP_EXPLAIN_STATEMENT_ID + "', 'ALL'))"; + List queryResults = jdbcOperations.query(getPlanSql, (rs, rowNum) -> rs.getString("PLAN_TABLE_OUTPUT")); + long estRowsValue = 0; + for (int rowNum = 5; rowNum < queryResults.size(); rowNum++) { + String resultRow = queryResults.get(rowNum); + estRowsValue = getEstRowsValue(resultRow); + if (estRowsValue != 0) { + break; + } + } + return estRowsValue; + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TruncateTableExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TruncateTableExists.java index 130562ff78..2f7f656828 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TruncateTableExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TruncateTableExists.java @@ -41,7 +41,7 @@ public class TruncateTableExists implements SqlCheckRule { @Override public SqlCheckRuleType getType() { - return SqlCheckRuleType.TRUNCATE_TBLE_EXISTS; + return SqlCheckRuleType.TRUNCATE_TABLE_EXISTS; } @Override diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/sqlcheck/OracleSqlCheckerTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/sqlcheck/OracleSqlCheckerTest.java index 22e3175570..ad5474ea9a 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/sqlcheck/OracleSqlCheckerTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/sqlcheck/OracleSqlCheckerTest.java @@ -1221,7 +1221,7 @@ public void check_truncateTbl_violationGenerated() { null, Collections.singletonList(new TruncateTableExists())); List actual = sqlChecker.check(toOffsetString(sqls), null); - SqlCheckRuleType type = SqlCheckRuleType.TRUNCATE_TBLE_EXISTS; + SqlCheckRuleType type = SqlCheckRuleType.TRUNCATE_TABLE_EXISTS; CheckViolation c1 = new CheckViolation(sqls[1], 1, 0, 0, 17, type, new Object[] {}); List expect = Collections.singletonList(c1); From 274f7e6083435d83125948ea90da511346b90665 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Thu, 31 Oct 2024 10:12:02 +0800 Subject: [PATCH 015/118] feat(pl): support editing pl sql for ob mysql (#3661) * feat(databaseChange): support editing pl sql for ob mysql * add IF EXISTS for drop procedure in ob mysql * make stopOnError true in editing ob mysql pl * use stringUtil rather than visitor to obtain temp pl * minimize length of lock key * support trigger and function * modify code format * Test two cases: the normal case and the case where the temporary pl already existed * modify code format * modify code format * modify code format * created DBPLModifyHelper to manage related methods * remove redundant code * modify code format * modify code format * add controller and service for modify pl * remove redundant code * remove redundant code * remove redundant code * remove redundant code * add integration test * removed @lazy * DBPLModifyHelper aggregate ConnectSessionService to execute the wrapped edit pl sql * modify according to commend * modify according to commend * modify according to commend * modify according to comments * modify code format --- .../odc/service/db/DBPLModifyHelperTest.java | 286 ++++++++++++++++++ .../web/controller/v2/DBPLController.java | 13 + .../odc/service/db/DBPLModifyHelper.java | 160 ++++++++++ .../odc/service/db/model/EditPLReq.java | 39 +++ .../odc/service/db/model/EditPLResp.java | 46 +++ .../session/ConnectConsoleService.java | 2 +- .../session/model/SqlAsyncExecuteResp.java | 1 + 7 files changed, 546 insertions(+), 1 deletion(-) create mode 100644 server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBPLModifyHelperTest.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLReq.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLResp.java diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBPLModifyHelperTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBPLModifyHelperTest.java new file mode 100644 index 0000000000..3e2994daf1 --- /dev/null +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBPLModifyHelperTest.java @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db; + +import static org.hibernate.validator.internal.util.Contracts.assertNotNull; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.jdbc.BadSqlGrammarException; + +import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.TestConnectionUtil; +import com.oceanbase.odc.common.util.VersionUtils; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionConstants; +import com.oceanbase.odc.core.session.ConnectionSessionUtil; +import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.core.shared.exception.BadRequestException; +import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor; +import com.oceanbase.odc.core.sql.split.SqlCommentProcessor; +import com.oceanbase.odc.service.db.model.EditPLReq; +import com.oceanbase.odc.service.db.model.EditPLResp; +import com.oceanbase.odc.service.session.ConnectSessionService; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/10/18 10:53 + * @since: 4.3.3 + */ +public class DBPLModifyHelperTest extends ServiceTestEnv { + private static final String ODC_TEST_PROCEDURE = "ODC_TEST_PROCEDURE"; + private static final String ODC_TEST_FUNCTION = "ODC_TEST_FUNCTION"; + private static final String ODC_TEST_TRIGGER = "ODC_TEST_TRIGGER"; + private static final String ODC_TEST_TRIGGER_TABLE = "ODC_TEST_TRIGGER_TABLE"; + + @MockBean + private ConnectSessionService sessionService; + + @Autowired + private DBPLModifyHelper dbplModifyHelper; + + + @BeforeClass + public static void setUp() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SyncJdbcExecutor syncJdbcExecutor = testConnectionSession.getSyncJdbcExecutor( + ConnectionSessionConstants.CONSOLE_DS_KEY); + // prepare test environment for procedure + executeDropPLSql(DBObjectType.PROCEDURE, ODC_TEST_PROCEDURE); + executeDropPLSql(DBObjectType.PROCEDURE, DBPLModifyHelper.ODC_TEMPORARY_PROCEDURE); + String createTestProcedure = "CREATE PROCEDURE " + ODC_TEST_PROCEDURE + "(IN num INT, OUT square INT)\n" + + "BEGIN\n" + + " SET square = num * num;\n" + + "END"; + syncJdbcExecutor.execute(createTestProcedure); + // prepare test environment for function + executeDropPLSql(DBObjectType.FUNCTION, ODC_TEST_FUNCTION); + executeDropPLSql(DBObjectType.FUNCTION, DBPLModifyHelper.ODC_TEMPORARY_FUNCTION); + String createTestFunction = "CREATE FUNCTION " + ODC_TEST_FUNCTION + "(num INT) \n" + + "RETURNS INT\n" + + "BEGIN\n" + + " RETURN num * num * num;\n" + + "END"; + syncJdbcExecutor.execute(createTestFunction); + // prepare test environment for trigger + executeDropPLSql(DBObjectType.TRIGGER, ODC_TEST_TRIGGER); + executeDropPLSql(DBObjectType.TRIGGER, DBPLModifyHelper.ODC_TEMPORARY_TRIGGER); + executeDropPLSql(DBObjectType.TABLE, ODC_TEST_TRIGGER_TABLE); + String createTestTable = "CREATE TABLE " + ODC_TEST_TRIGGER_TABLE + "(\n" + + " id INT AUTO_INCREMENT PRIMARY KEY,\n" + + " value INT\n" + + ");"; + syncJdbcExecutor.execute(createTestTable); + String createTestTrigger = "CREATE TRIGGER " + ODC_TEST_TRIGGER + "\n" + + "BEFORE INSERT ON " + ODC_TEST_TRIGGER_TABLE + "\n" + + "FOR EACH ROW\n" + + "BEGIN\n" + + " SET NEW.value = NEW.value * 1;\n" + + "END"; + syncJdbcExecutor.execute(createTestTrigger); + } + + + @AfterClass + public static void clear() throws Exception { + // clear test environment for procedure + executeDropPLSql(DBObjectType.PROCEDURE, ODC_TEST_PROCEDURE); + executeDropPLSql(DBObjectType.PROCEDURE, DBPLModifyHelper.ODC_TEMPORARY_PROCEDURE); + // clear test environment for function + executeDropPLSql(DBObjectType.FUNCTION, ODC_TEST_FUNCTION); + executeDropPLSql(DBObjectType.FUNCTION, DBPLModifyHelper.ODC_TEMPORARY_FUNCTION); + // clear test environment for trigger + executeDropPLSql(DBObjectType.TRIGGER, ODC_TEST_TRIGGER); + executeDropPLSql(DBObjectType.TRIGGER, DBPLModifyHelper.ODC_TEMPORARY_TRIGGER); + executeDropPLSql(DBObjectType.TABLE, ODC_TEST_TRIGGER_TABLE); + } + + @Before + public void mock() { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SqlCommentProcessor sqlCommentProcessor = new SqlCommentProcessor(DialectType.OB_MYSQL, true, true, true); + testConnectionSession.setAttribute(ConnectionSessionConstants.SQL_COMMENT_PROCESSOR_KEY, sqlCommentProcessor); + String sessionId = testConnectionSession.getId(); + Mockito.when(sessionService.nullSafeGet(sessionId, true)).thenReturn(testConnectionSession); + Mockito.when(sessionService.nullSafeGet(sessionId)).thenReturn(testConnectionSession); + } + + @Test + public void editProcedureForOBMysql_normal_successResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + String editTestProcedure = "CREATE PROCEDURE " + ODC_TEST_PROCEDURE + "(IN num1 INT, OUT square1 INT)\n" + + "BEGIN\n" + + " SET square1 = num1 * num1;\n" + + "END"; + EditPLResp editPLResp = executeEditPL(testConnectionSession, editTestProcedure, ODC_TEST_PROCEDURE, + DBObjectType.PROCEDURE); + assertNotNull(editPLResp); + assertFalse(editPLResp.isApprovalRequired()); + assertNull(editPLResp.getErrorMessage()); + } + + + @Test + public void editProcedureForOBMysql_odcTempProcedureHaveExisted_failResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SyncJdbcExecutor syncJdbcExecutor = testConnectionSession.getSyncJdbcExecutor( + ConnectionSessionConstants.CONSOLE_DS_KEY); + String createTempProcedure = "CREATE PROCEDURE " + DBPLModifyHelper.ODC_TEMPORARY_PROCEDURE + + "(IN num INT, OUT square INT)\n" + + "BEGIN\n" + + " SET square = num * num;\n" + + "END"; + syncJdbcExecutor.execute(createTempProcedure); + String editTestProcedure = "CREATE PROCEDURE " + ODC_TEST_PROCEDURE + "(IN num1 INT, OUT square1 INT)\n" + + "BEGIN\n" + + " SET square1 = num1 * num1;\n" + + "END"; + EditPLResp editPLResp = executeEditPL(testConnectionSession, editTestProcedure, ODC_TEST_PROCEDURE, + DBObjectType.PROCEDURE); + assertNotNull(editPLResp); + assertFalse(editPLResp.isApprovalRequired()); + assertNotNull(editPLResp.getErrorMessage()); + executeDropPLSql(DBObjectType.PROCEDURE, DBPLModifyHelper.ODC_TEMPORARY_PROCEDURE); + } + + @Test + public void editFunctionForOBMysql_normal_successResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + String editTestFunction = "CREATE FUNCTION " + ODC_TEST_FUNCTION + "(num1 INT) \n" + + "RETURNS INT\n" + + "BEGIN\n" + + " RETURN num1 * num1 * num1;\n" + + "END"; + EditPLResp editPLResp = executeEditPL(testConnectionSession, editTestFunction, ODC_TEST_FUNCTION, + DBObjectType.FUNCTION); + assertNotNull(editPLResp); + assertFalse(editPLResp.isApprovalRequired()); + assertNull(editPLResp.getErrorMessage()); + } + + @Test + public void editFunctionForOBMysql_odcTempFunctionHaveExisted_failResultResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SyncJdbcExecutor syncJdbcExecutor = testConnectionSession.getSyncJdbcExecutor( + ConnectionSessionConstants.CONSOLE_DS_KEY); + String createODCTempFunction = + "CREATE FUNCTION " + DBPLModifyHelper.ODC_TEMPORARY_FUNCTION + "(num INT) \n" + + "RETURNS INT\n" + + "BEGIN\n" + + " RETURN num * num * num;\n" + + "END"; + syncJdbcExecutor.execute(createODCTempFunction); + String editTestFunction = "CREATE FUNCTION " + ODC_TEST_FUNCTION + "(num1 INT) \n" + + "RETURNS INT\n" + + "BEGIN\n" + + " RETURN num1 * num1 * num1;\n" + + "END"; + EditPLResp editPLResp = executeEditPL(testConnectionSession, editTestFunction, ODC_TEST_FUNCTION, + DBObjectType.FUNCTION); + assertNotNull(editPLResp); + assertFalse(editPLResp.isApprovalRequired()); + assertNotNull(editPLResp.getErrorMessage()); + executeDropPLSql(DBObjectType.FUNCTION, DBPLModifyHelper.ODC_TEMPORARY_FUNCTION); + } + + @Test + public void editTriggerForOBMysql_normal_successResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + String editTestTrigger = "CREATE TRIGGER " + ODC_TEST_TRIGGER + "\n" + + "BEFORE INSERT ON " + ODC_TEST_TRIGGER_TABLE + "\n" + + "FOR EACH ROW\n" + + "BEGIN\n" + + " SET NEW.value = NEW.value * 2;\n" + + "END"; + if (VersionUtils.isLessThan(ConnectionSessionUtil.getVersion(testConnectionSession), + DBPLModifyHelper.OB_VERSION_SUPPORT_MULTIPLE_SAME_TRIGGERS)) { + assertThrows(BadRequestException.class, + () -> executeEditPL(testConnectionSession, editTestTrigger, ODC_TEST_TRIGGER, + DBObjectType.TRIGGER)); + } else { + EditPLResp editPLResp = executeEditPL(testConnectionSession, editTestTrigger, ODC_TEST_TRIGGER, + DBObjectType.TRIGGER); + assertNotNull(editPLResp); + assertFalse(editPLResp.isApprovalRequired()); + assertNull(editPLResp.getErrorMessage()); + } + } + + @Test + public void editTriggerForOBMysql_odcTempTriggerHaveExisted_failResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SyncJdbcExecutor syncJdbcExecutor = testConnectionSession.getSyncJdbcExecutor( + ConnectionSessionConstants.CONSOLE_DS_KEY); + String createODCTempTrigger = + "CREATE TRIGGER " + DBPLModifyHelper.ODC_TEMPORARY_TRIGGER + "\n" + + "BEFORE UPDATE ON " + ODC_TEST_TRIGGER_TABLE + "\n" + + "FOR EACH ROW\n" + + "BEGIN\n" + + " SET NEW.value = NEW.value * 2;\n" + + "END"; + syncJdbcExecutor.execute(createODCTempTrigger); + String editTestTrigger = + "CREATE TRIGGER " + ODC_TEST_TRIGGER + "\n" + + "BEFORE INSERT ON " + ODC_TEST_TRIGGER_TABLE + "\n" + + "FOR EACH ROW\n" + + "BEGIN\n" + + " SET NEW.value = NEW.value * 2;\n" + + "END"; + + if (VersionUtils.isLessThan(ConnectionSessionUtil.getVersion(testConnectionSession), + DBPLModifyHelper.OB_VERSION_SUPPORT_MULTIPLE_SAME_TRIGGERS)) { + assertThrows(BadRequestException.class, + () -> executeEditPL(testConnectionSession, editTestTrigger, ODC_TEST_TRIGGER, + DBObjectType.TRIGGER)); + } else { + assertThrows(BadSqlGrammarException.class, + () -> executeEditPL(testConnectionSession, editTestTrigger, ODC_TEST_TRIGGER, + DBObjectType.TRIGGER)); + } + executeDropPLSql(DBObjectType.TRIGGER, DBPLModifyHelper.ODC_TEMPORARY_TRIGGER); + executeDropPLSql(DBObjectType.TABLE, ODC_TEST_TRIGGER_TABLE); + } + + + private EditPLResp executeEditPL(ConnectionSession testConnectionSession, String editTestPL, String plName, + DBObjectType plType) + throws Exception { + EditPLReq editPLReq = new EditPLReq(); + editPLReq.setSql(editTestPL); + editPLReq.setObjectType(plType); + editPLReq.setObjectName(plName); + return dbplModifyHelper.editPL(testConnectionSession.getId(), editPLReq); + } + + private static void executeDropPLSql(DBObjectType plType, String plName) { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SyncJdbcExecutor syncJdbcExecutor = testConnectionSession.getSyncJdbcExecutor( + ConnectionSessionConstants.CONSOLE_DS_KEY); + String dropTestProcedure = "DROP " + plType + " IF EXISTS " + plName; + syncJdbcExecutor.execute(dropTestProcedure); + } +} diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBPLController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBPLController.java index 39052b30d7..469783fbbd 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBPLController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBPLController.java @@ -27,6 +27,7 @@ import com.oceanbase.odc.service.common.response.Responses; import com.oceanbase.odc.service.common.response.SuccessResponse; import com.oceanbase.odc.service.common.util.SidUtils; +import com.oceanbase.odc.service.db.DBPLModifyHelper; import com.oceanbase.odc.service.db.DBPLService; import com.oceanbase.odc.service.db.model.CallFunctionReq; import com.oceanbase.odc.service.db.model.CallFunctionResp; @@ -34,6 +35,8 @@ import com.oceanbase.odc.service.db.model.CallProcedureResp; import com.oceanbase.odc.service.db.model.CompileResult; import com.oceanbase.odc.service.db.model.DBMSOutput; +import com.oceanbase.odc.service.db.model.EditPLReq; +import com.oceanbase.odc.service.db.model.EditPLResp; import com.oceanbase.odc.service.db.model.PLIdentity; import com.oceanbase.odc.service.session.ConnectSessionService; import com.oceanbase.odc.service.state.model.StateName; @@ -50,6 +53,8 @@ public class DBPLController { private DBPLService plService; @Autowired private ConnectSessionService sessionService; + @Autowired + private DBPLModifyHelper dBPLmodifyHelper; @ApiOperation(value = "compile", notes = "compile a pl object") @RequestMapping(value = "/compile/{sid}", method = RequestMethod.POST) @@ -110,4 +115,12 @@ public SuccessResponse getNameAndType(@PathVariable String sid, @Req return Responses.ok(PLIdentity.of(identity.getType(), identity.getName())); } + @ApiOperation(value = "editPL", notes = "edit DDL of a pl object") + @RequestMapping(value = "/editPL/{sid}", method = RequestMethod.POST) + @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sid") + public SuccessResponse editPL(@PathVariable String sid, @RequestBody EditPLReq editPLReq) + throws Exception { + return Responses.ok(this.dBPLmodifyHelper.editPL(SidUtils.getSessionId(sid), editPLReq)); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java new file mode 100644 index 0000000000..c08a07de84 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.integration.jdbc.lock.JdbcLockRegistry; +import org.springframework.stereotype.Component; +import org.springframework.validation.annotation.Validated; + +import com.oceanbase.odc.common.util.VersionUtils; +import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionUtil; +import com.oceanbase.odc.core.shared.constant.ErrorCodes; +import com.oceanbase.odc.core.shared.exception.BadRequestException; +import com.oceanbase.odc.core.shared.exception.ConflictException; +import com.oceanbase.odc.core.sql.execute.model.SqlExecuteStatus; +import com.oceanbase.odc.core.sql.split.SqlCommentProcessor; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.db.model.EditPLReq; +import com.oceanbase.odc.service.db.model.EditPLResp; +import com.oceanbase.odc.service.session.ConnectConsoleService; +import com.oceanbase.odc.service.session.ConnectSessionService; +import com.oceanbase.odc.service.session.model.AsyncExecuteResultResp; +import com.oceanbase.odc.service.session.model.SqlAsyncExecuteReq; +import com.oceanbase.odc.service.session.model.SqlAsyncExecuteResp; +import com.oceanbase.odc.service.session.model.SqlExecuteResult; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/10/17 18:59 + * @since: 4.3.3 + */ +@Validated +@Component +@SkipAuthorize("permission check inside") +public class DBPLModifyHelper { + public static final String ODC_TEMPORARY_PROCEDURE = "_ODC_TEMPORARY_PROCEDURE"; + public static final String ODC_TEMPORARY_TRIGGER = "_ODC_TEMPORARY_TRIGGER"; + public static final String ODC_TEMPORARY_FUNCTION = "_ODC_TEMPORARY_FUNCTION"; + public static final String OB_VERSION_SUPPORT_MULTIPLE_SAME_TRIGGERS = "4.2"; + public static final int LOCK_TIMEOUT_SECONDS = 3; + @Autowired + private ConnectConsoleService connectConsoleService; + @Autowired + private JdbcLockRegistry jdbcLockRegistry; + @Autowired + private ConnectSessionService sessionService; + + public EditPLResp editPL(@NotNull String sessionId, @NotNull @Valid EditPLReq editPLReq) + throws Exception { + DBObjectType plType = editPLReq.getObjectType(); + switch (plType) { + case PROCEDURE: + return executeWrappedEditPL(sessionId, editPLReq, DBObjectType.PROCEDURE, ODC_TEMPORARY_PROCEDURE); + case FUNCTION: + return executeWrappedEditPL(sessionId, editPLReq, DBObjectType.FUNCTION, ODC_TEMPORARY_FUNCTION); + case TRIGGER: + if (VersionUtils.isLessThan( + ConnectionSessionUtil.getVersion(sessionService.nullSafeGet(sessionId, true)), + OB_VERSION_SUPPORT_MULTIPLE_SAME_TRIGGERS)) { + throw new BadRequestException( + "editing trigger in mysql mode is not supported in ob version less than " + + OB_VERSION_SUPPORT_MULTIPLE_SAME_TRIGGERS); + } + return executeWrappedEditPL(sessionId, editPLReq, DBObjectType.TRIGGER, ODC_TEMPORARY_TRIGGER); + default: + throw new IllegalArgumentException( + String.format("the pl type %s of editing procedure is not supported", plType)); + } + } + + private EditPLResp executeWrappedEditPL(String sessionId, EditPLReq editPLReq, + DBObjectType plType, String tempPlName) throws Exception { + String plName = editPLReq.getObjectName(); + String editPLSql = editPLReq.getSql(); + String tempPLSql = editPLSql.replaceFirst(plName, tempPlName); + StringBuilder wrappedSqlBuilder = new StringBuilder(); + ConnectionSession connectionSession = sessionService.nullSafeGet(sessionId, true); + SqlCommentProcessor processor = ConnectionSessionUtil.getSqlCommentProcessor(connectionSession); + String delimiter = processor.getDelimiter(); + wrappedSqlBuilder.append("DELIMITER $$\n") + .append(tempPLSql).append(" $$\n") + .append("drop ").append(plType).append(" if exists ").append(tempPlName).append(" $$\n") + .append("drop ").append(plType).append(" if exists ").append(plName).append(" $$\n") + .append(editPLSql).append(" $$\n") + .append("DELIMITER " + delimiter); + String wrappedSql = wrappedSqlBuilder.toString(); + SqlAsyncExecuteReq sqlAsyncExecuteReq = new SqlAsyncExecuteReq(); + sqlAsyncExecuteReq.setSql(wrappedSql); + sqlAsyncExecuteReq.setSplit(true); + sqlAsyncExecuteReq.setContinueExecutionOnError(false); + + Lock editPLLock = obtainEditPLLock(connectionSession, plType); + if (!editPLLock.tryLock(LOCK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + throw new ConflictException(ErrorCodes.ResourceModifying, "Can not acquire jdbc lock"); + } + try { + SqlAsyncExecuteResp sqlAsyncExecuteResp = connectConsoleService.streamExecute(sessionId, sqlAsyncExecuteReq, + true); + EditPLResp editPLResp = new EditPLResp(); + editPLResp.setWrappedSql(wrappedSql); + editPLResp.setSqls(sqlAsyncExecuteResp.getSqls()); + editPLResp.setUnauthorizedDBResources(sqlAsyncExecuteResp.getUnauthorizedDBResources()); + editPLResp.setViolatedRules(sqlAsyncExecuteResp.getViolatedRules()); + editPLResp.setApprovalRequired(sqlAsyncExecuteResp.isApprovalRequired()); + if (editPLResp.isApprovalRequired()) { + return editPLResp; + } + AsyncExecuteResultResp moreResults; + List results = new ArrayList<>(); + do { + moreResults = connectConsoleService.getMoreResults(sessionId, + sqlAsyncExecuteResp.getRequestId()); + results.addAll(moreResults.getResults()); + } while (!moreResults.isFinished()); + for (SqlExecuteResult result : results) { + if (result.getStatus() != SqlExecuteStatus.SUCCESS) { + editPLResp.setErrorMessage(result.getTrack()); + return editPLResp; + } + } + return editPLResp; + } finally { + editPLLock.unlock(); + } + } + + private Lock obtainEditPLLock(@NotNull ConnectionSession connectionSession, @NonNull DBObjectType plType) { + ConnectionConfig connConfig = + (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(connectionSession); + Long dataSourceId = connConfig.getId(); + return jdbcLockRegistry.obtain(String.format("%s-%d", plType, dataSourceId)); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLReq.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLReq.java new file mode 100644 index 0000000000..7c0fba2fa0 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLReq.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db.model; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +import lombok.Data; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/10/18 13:00 + * @since: 4.3.3 + */ +@Data +public class EditPLReq { + @NotBlank + private String sql; + @NotBlank + private String objectName; + @NotNull + private DBObjectType objectType; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLResp.java new file mode 100644 index 0000000000..2bedcf9690 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLResp.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db.model; + +import java.util.List; + +import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; +import com.oceanbase.odc.service.regulation.ruleset.model.Rule; +import com.oceanbase.odc.service.session.model.SqlTuplesWithViolation; + +import lombok.Data; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/10/18 14:11 + * @since: 4.3.3 + */ +@Data +public class EditPLResp { + // if pre-check is not successful ,sql confirmation window needs to display wrapped Sql instead of + // the original sql.Because what really executes is wrapped Sql + private String wrappedSql; + // if shouldIntercepted is true，it indicates sql pre-check does not pass. sql confirmation window + // needs to be opened + private boolean approvalRequired; + private List violatedRules; + private List sqls; + private List unauthorizedDBResources; + // if none, the modification succeeded.If not none, the modification failed.it indicates the error + // message + private String errorMessage; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java index 6ea4a55956..27f53fcec9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java @@ -287,6 +287,7 @@ public SqlAsyncExecuteResp streamExecute(@NotNull String sessionId, .collect(Collectors.toList()); try { if (!sqlInterceptService.preHandle(request, response, connectionSession, executeContext)) { + response.setApprovalRequired(true); return response; } } finally { @@ -620,5 +621,4 @@ private Integer checkQueryLimit(Integer queryLimit) { } return queryLimit; } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java index 8086eb6646..79dc05178d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java @@ -39,6 +39,7 @@ public class SqlAsyncExecuteResp { private List sqls; private List unauthorizedDBResources; private boolean logicalSql; + private boolean approvalRequired; public SqlAsyncExecuteResp(boolean logicalSql) { this.logicalSql = logicalSql; From 918c208a1208924e13dafc7e0659a2fa7f1b3d7e Mon Sep 17 00:00:00 2001 From: LioRoger Date: Thu, 31 Oct 2024 11:24:40 +0800 Subject: [PATCH 016/118] feat(task): refactor task getStatus interface return TaskStatus instead of JobStatus (#3766) --- .../com/oceanbase/odc/service/task/Task.java | 3 +- .../odc/service/task/base/BaseTask.java | 18 +-- .../task/executor/DefaultTaskResult.java | 4 +- .../service/task/executor/TaskMonitor.java | 4 +- .../odc/service/task/executor/TaskResult.java | 4 +- .../DefaultJobProcessUpdateListener.java | 1 - .../task/service/StdTaskFrameworkService.java | 138 ++++++++---------- .../odc/service/task/state/JobStatusFsm.java | 57 ++++++++ .../service/task/status/JobStatusFsmTest.java | 55 +++++++ 9 files changed, 193 insertions(+), 91 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/state/JobStatusFsm.java create mode 100644 server/odc-service/src/test/java/com/oceanbase/odc/service/task/status/JobStatusFsmTest.java diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/Task.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/Task.java index b3b74b3b86..b4f89b0251 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/Task.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/Task.java @@ -17,6 +17,7 @@ import java.util.Map; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.enums.JobStatus; @@ -62,7 +63,7 @@ public interface Task { * * @return {@link JobStatus} */ - JobStatus getStatus(); + TaskStatus getStatus(); /** * Get task result diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java index f68e597a2e..201e410d57 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java @@ -21,12 +21,12 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.caller.DefaultJobContext; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.executor.TaskMonitor; import com.oceanbase.odc.service.task.util.CloudObjectStorageServiceBuilder; import com.oceanbase.odc.service.task.util.JobUtils; @@ -44,7 +44,7 @@ public abstract class BaseTask implements Task { private final AtomicBoolean closed = new AtomicBoolean(false); private JobContext context; private Map jobParameters; - private volatile JobStatus status = JobStatus.PREPARING; + private volatile TaskStatus status = TaskStatus.PREPARING; private CloudObjectStorageService cloudObjectStorageService; @Getter @@ -68,16 +68,16 @@ public void start(JobContext context) { this.taskMonitor = new TaskMonitor(this, cloudObjectStorageService); try { doInit(context); - updateStatus(JobStatus.RUNNING); + updateStatus(TaskStatus.RUNNING); taskMonitor.monitor(); if (doStart(context)) { - updateStatus(JobStatus.DONE); + updateStatus(TaskStatus.DONE); } else { - updateStatus(JobStatus.FAILED); + updateStatus(TaskStatus.FAILED); } } catch (Throwable e) { log.warn("Task failed, id={}.", getJobId(), e); - updateStatus(JobStatus.FAILED); + updateStatus(TaskStatus.FAILED); } finally { close(); } @@ -91,7 +91,7 @@ public boolean stop() { } else { doStop(); // doRefresh cannot execute if update status to 'canceled'. - updateStatus(JobStatus.CANCELING); + updateStatus(TaskStatus.CANCELED); } return true; } catch (Throwable e) { @@ -145,7 +145,7 @@ protected CloudObjectStorageService getCloudObjectStorageService() { } @Override - public JobStatus getStatus() { + public TaskStatus getStatus() { return status; } @@ -154,7 +154,7 @@ public JobContext getJobContext() { return context; } - protected void updateStatus(JobStatus status) { + protected void updateStatus(TaskStatus status) { log.info("Update task status, id={}, status={}.", getJobId(), status); this.status = status; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResult.java index 28bb7a3d5b..0d4849bc6d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResult.java @@ -17,7 +17,7 @@ import java.util.Map; -import com.oceanbase.odc.service.task.enums.JobStatus; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.task.schedule.JobIdentity; import lombok.Data; @@ -32,7 +32,7 @@ public class DefaultTaskResult implements TaskResult { private JobIdentity jobIdentity; - private JobStatus status; + private TaskStatus status; private String resultJson; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskMonitor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskMonitor.java index c6cb1b8687..e33735e24b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskMonitor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskMonitor.java @@ -27,6 +27,7 @@ import org.apache.commons.lang3.StringUtils; import com.oceanbase.odc.common.util.ObjectUtil; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.core.task.TaskThreadFactory; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; import com.oceanbase.odc.service.task.base.BaseTask; @@ -34,7 +35,6 @@ import com.oceanbase.odc.service.task.constants.JobConstants; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.constants.JobServerUrls; -import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.executor.logger.LogBizImpl; import com.oceanbase.odc.service.task.util.JobUtils; @@ -226,7 +226,7 @@ private void uploadLogFileToCloudStorage(DefaultTaskResult finalResult) { } private void reportTaskResultWithRetry(DefaultTaskResult result, int retries) { - if (result.getStatus() == JobStatus.DONE) { + if (result.getStatus() == TaskStatus.DONE) { result.setProgress(100.0); } int retryTimes = 0; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java index d823afd0c4..6b42652bc1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java @@ -17,7 +17,7 @@ import java.util.Map; -import com.oceanbase.odc.service.task.enums.JobStatus; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.task.schedule.JobIdentity; /** @@ -29,7 +29,7 @@ public interface TaskResult { JobIdentity getJobIdentity(); - JobStatus getStatus(); + TaskStatus getStatus(); String getResultJson(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java index 643428f3b7..59e9780f73 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java @@ -56,7 +56,6 @@ public void onEvent(DefaultJobProcessUpdateEvent event) { updateScheduleTaskStatus(taskEntity.getId(), TaskStatus.RUNNING, TaskStatus.PREPARING); } }); - } private void updateScheduleTaskStatus(Long id, TaskStatus status, TaskStatus previousStatus) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java index 0b62784e77..a2390a8c47 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java @@ -48,7 +48,6 @@ import com.oceanbase.odc.common.event.EventPublisher; import com.oceanbase.odc.common.jpa.SpecificationUtil; import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.common.security.SensitiveDataUtils; import com.oceanbase.odc.common.trace.TraceContextHolder; import com.oceanbase.odc.common.util.ExceptionUtils; import com.oceanbase.odc.common.util.StringUtils; @@ -58,6 +57,7 @@ import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.metadb.resource.ResourceEntity; import com.oceanbase.odc.metadb.resource.ResourceRepository; @@ -87,6 +87,7 @@ import com.oceanbase.odc.service.task.processor.LogicalDBChangeResultProcessor; import com.oceanbase.odc.service.task.schedule.JobDefinition; import com.oceanbase.odc.service.task.schedule.JobIdentity; +import com.oceanbase.odc.service.task.state.JobStatusFsm; import com.oceanbase.odc.service.task.util.JobDateUtils; import com.oceanbase.odc.service.task.util.JobPropertiesUtils; import com.oceanbase.odc.service.task.util.TaskExecutorClient; @@ -143,6 +144,9 @@ public class StdTaskFrameworkService implements TaskFrameworkService { @Autowired private LoadDataResultProcessor loadDataResultProcessor; + // default impl + private JobStatusFsm jobStatusFsm = new JobStatusFsm(); + @Override public JobEntity find(Long id) { return jobRepository.findByIdNative(id) @@ -325,7 +329,7 @@ public void handleResult(TaskResult taskResult) { log.warn("Job identity is null"); return; } - if (taskResult.getStatus() == JobStatus.CANCELED) { + if (taskResult.getStatus() == TaskStatus.CANCELED) { log.warn("Job is canceled by odc server, this result is ignored."); return; } @@ -348,70 +352,7 @@ public void handleResult(TaskResult taskResult) { log.warn("Update executor endpoint failed, jobId={}", je.getId()); } } - // TODO: update task entity only when progress changed - int rows = updateTaskResult(taskResult, je); - tryReleaseResource(je, taskResult.getStatus().isTerminated()); - if (rows > 0) { - taskResultPublisherExecutor - .execute(() -> publisher.publishEvent(new DefaultJobProcessUpdateEvent(taskResult))); - - if (publisher != null && taskResult.getStatus() != null && taskResult.getStatus().isTerminated()) { - taskResultPublisherExecutor.execute(() -> publisher - .publishEvent(new JobTerminateEvent(taskResult.getJobIdentity(), taskResult.getStatus()))); - - if (taskResult.getStatus() == JobStatus.FAILED) { - AlarmUtils.alarm(AlarmEventNames.TASK_EXECUTION_FAILED, - MessageFormat.format("Job execution failed, jobId={0}, resultJson={1}", - taskResult.getJobIdentity().getId(), - SensitiveDataUtils.mask(taskResult.getResultJson()))); - } - } - } - } - - @Override - public void refreshResult(Long id) { - taskResultPullerExecutor.execute(() -> { - try { - doRefreshResult(id); - } catch (Exception e) { - log.warn("Refresh job result failed, jobId={}, causeReason={}", - id, ExceptionUtils.getRootCauseReason(e)); - } - }); - } - - @Override - public boolean refreshLogMeta(Long id) { - JobEntity je = find(id); - // CANCELING is also a state within the running phase - if (JobStatus.RUNNING != je.getStatus() && JobStatus.CANCELING != je.getStatus()) { - log.warn("Job is not running, don't need to refresh log meta, jobId={}, currentStatus={}", id, - je.getStatus()); - return true; - } - try { - String executorEndpoint = executorEndpointManager.getExecutorEndpoint(je); - DefaultTaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); - - if (je.getRunMode().isK8s() && MapUtils.isEmpty(result.getLogMetadata())) { - log.info("Refresh log failed due to log have not uploaded, jobId={}, currentStatus={}", je.getId(), - je.getStatus()); - return false; - } - saveOrUpdateLogMetadata(result, je.getId(), je.getStatus()); - if ("DLM".equals(je.getJobType())) { - dlmResultProcessor.process(result); - } else if (StringUtils.equalsIgnoreCase("LogicalDatabaseChange", je.getJobType())) { - logicalDBChangeResultProcessor.process(result); - } else if (StringUtils.equalsIgnoreCase("LOAD_DATA", je.getJobType())) { - loadDataResultProcessor.process(result); - } - return true; - } catch (Exception exception) { - log.warn("Refresh log meta failed,errorMsg={}", exception.getMessage()); - return false; - } + handleTaskResultInner(je, taskResult); } private void doRefreshResult(Long id) throws JobException { @@ -424,7 +365,7 @@ private void doRefreshResult(Long id) throws JobException { String executorEndpoint = executorEndpointManager.getExecutorEndpoint(je); DefaultTaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); - if (result.getStatus() == JobStatus.PREPARING) { + if (result.getStatus() == TaskStatus.PREPARING) { log.info("Job is preparing, ignore refresh, jobId={}, currentStatus={}", id, result.getStatus()); return; } @@ -452,12 +393,16 @@ private void doRefreshResult(Long id) throws JobException { je.getId(), je.getStatus()); return; } + handleTaskResultInner(je, result); + } - int rows = updateTaskResult(result, je); + protected void handleTaskResultInner(JobEntity jobEntity, TaskResult result) { + JobStatus expectedJobStatus = jobStatusFsm.determinateJobStatus(jobEntity.getStatus(), result.getStatus()); + int rows = updateTaskResult(result, jobEntity, expectedJobStatus); // release resource - tryReleaseResource(je, result.getStatus().isTerminated()); + tryReleaseResource(jobEntity, expectedJobStatus.isTerminated()); if (rows == 0) { - log.warn("Update task result failed, the job may finished or deleted already, jobId={}", id); + log.warn("Update task result failed, the job may finished or deleted already, jobId={}", jobEntity.getId()); return; } taskResultPublisherExecutor @@ -465,10 +410,10 @@ private void doRefreshResult(Long id) throws JobException { if (publisher != null && result.getStatus() != null && result.getStatus().isTerminated()) { taskResultPublisherExecutor.execute(() -> publisher - .publishEvent(new JobTerminateEvent(result.getJobIdentity(), result.getStatus()))); + .publishEvent(new JobTerminateEvent(result.getJobIdentity(), expectedJobStatus))); // TODO maybe we can destroy the pod there. - if (result.getStatus() == JobStatus.FAILED) { + if (result.getStatus() == TaskStatus.FAILED) { AlarmUtils.alarm(AlarmEventNames.TASK_EXECUTION_FAILED, MessageFormat.format("Job execution failed, jobId={0}", result.getJobIdentity().getId())); @@ -476,6 +421,51 @@ private void doRefreshResult(Long id) throws JobException { } } + @Override + public void refreshResult(Long id) { + taskResultPullerExecutor.execute(() -> { + try { + doRefreshResult(id); + } catch (Exception e) { + log.warn("Refresh job result failed, jobId={}, causeReason={}", + id, ExceptionUtils.getRootCauseReason(e)); + } + }); + } + + @Override + public boolean refreshLogMeta(Long id) { + JobEntity je = find(id); + // CANCELING is also a state within the running phase + if (JobStatus.RUNNING != je.getStatus() && JobStatus.CANCELING != je.getStatus()) { + log.warn("Job is not running, don't need to refresh log meta, jobId={}, currentStatus={}", id, + je.getStatus()); + return true; + } + try { + String executorEndpoint = executorEndpointManager.getExecutorEndpoint(je); + DefaultTaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); + + if (je.getRunMode().isK8s() && MapUtils.isEmpty(result.getLogMetadata())) { + log.info("Refresh log failed due to log have not uploaded, jobId={}, currentStatus={}", je.getId(), + je.getStatus()); + return false; + } + saveOrUpdateLogMetadata(result, je.getId(), je.getStatus()); + if ("DLM".equals(je.getJobType())) { + dlmResultProcessor.process(result); + } else if (StringUtils.equalsIgnoreCase("LogicalDatabaseChange", je.getJobType())) { + logicalDBChangeResultProcessor.process(result); + } else if (StringUtils.equalsIgnoreCase("LOAD_DATA", je.getJobType())) { + loadDataResultProcessor.process(result); + } + return true; + } catch (Exception exception) { + log.warn("Refresh log meta failed,errorMsg={}", exception.getMessage()); + return false; + } + } + protected void tryReleaseResource(JobEntity jobEntity, boolean isJobDone) { // release resource if (isJobDone && TaskRunMode.K8S == jobEntity.getRunMode()) { @@ -524,7 +514,7 @@ private int updateExecutorEndpoint(Long id, String executorEndpoint, JobEntity c return jobRepository.updateExecutorEndpoint(id, executorEndpoint, currentJob.getStatus()); } - private int updateTaskResult(TaskResult taskResult, JobEntity currentJob) { + private int updateTaskResult(TaskResult taskResult, JobEntity currentJob, JobStatus expectedStatus) { JobEntity jse = new JobEntity(); if ("DLM".equals(currentJob.getJobType())) { dlmResultProcessor.process(taskResult); @@ -532,7 +522,7 @@ private int updateTaskResult(TaskResult taskResult, JobEntity currentJob) { logicalDBChangeResultProcessor.process(taskResult); } jse.setResultJson(taskResult.getResultJson()); - jse.setStatus(taskResult.getStatus()); + jse.setStatus(expectedStatus); jse.setProgressPercentage(taskResult.getProgress()); jse.setLastReportTime(JobDateUtils.getCurrentDate()); if (taskResult.getStatus() != null && taskResult.getStatus().isTerminated()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/state/JobStatusFsm.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/state/JobStatusFsm.java new file mode 100644 index 0000000000..f99162bfb5 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/state/JobStatusFsm.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.state; + +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.task.enums.JobStatus; + +/** + * default fsm for job status transfer according job current status and task status + * + * @author longpeng.zlp + * @date 2024/10/22 14:15 + */ +public class JobStatusFsm { + /** + * determinate job status by task status and current job status + * + * @param currentStatus + * @param taskStatus + * @return + */ + public JobStatus determinateJobStatus(JobStatus currentStatus, TaskStatus taskStatus) { + switch (taskStatus) { + // prepare is task init status, job should expected in running status + // running Job status is expected, if job = canceling, outside may set to cancel when task timeout + case PREPARING: + // running meaning task has running normally, job should expected in running status + case RUNNING: + return currentStatus; + // if task status is canceled, it must be canceled by job who owns it + case CANCELED: + return JobStatus.CANCELED; + // any task failed will cause the failed status of job + case ABNORMAL: + case FAILED: + return JobStatus.FAILED; + // task done will drive job status to done + case DONE: + return JobStatus.DONE; + default: + throw new IllegalStateException("status " + taskStatus + " not handled"); + } + } +} diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/status/JobStatusFsmTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/status/JobStatusFsmTest.java new file mode 100644 index 0000000000..195b2f30b7 --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/status/JobStatusFsmTest.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.status; + +import org.junit.Assert; +import org.junit.Test; + +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.task.enums.JobStatus; +import com.oceanbase.odc.service.task.state.JobStatusFsm; + +/** + * @author longpeng.zlp + * @date 2024/10/23 09:54 + */ +public class JobStatusFsmTest { + private final JobStatusFsm jobStatusFsm = new JobStatusFsm(); + + @Test + public void testJobStatusFsm() { + // check remaining + checkStatusRemain(TaskStatus.RUNNING); + checkStatusRemain(TaskStatus.PREPARING); + // check transfer + checkStatusTransfer(TaskStatus.CANCELED, JobStatus.CANCELED); + checkStatusTransfer(TaskStatus.DONE, JobStatus.DONE); + checkStatusTransfer(TaskStatus.FAILED, JobStatus.FAILED); + checkStatusTransfer(TaskStatus.ABNORMAL, JobStatus.FAILED); + } + + private void checkStatusRemain(TaskStatus remainStatus) { + for (JobStatus jobStatus : JobStatus.values()) { + Assert.assertEquals(jobStatus, jobStatusFsm.determinateJobStatus(jobStatus, remainStatus)); + } + } + + private void checkStatusTransfer(TaskStatus remainStatus, JobStatus targetStatus) { + for (JobStatus jobStatus : JobStatus.values()) { + Assert.assertEquals(targetStatus, jobStatusFsm.determinateJobStatus(jobStatus, remainStatus)); + } + } +} From 7cae30050ec3c5a3d02635a7f25ae0550d749946 Mon Sep 17 00:00:00 2001 From: IL MARE Date: Fri, 1 Nov 2024 18:14:56 +0800 Subject: [PATCH 017/118] feat(ob-sql-parser): make parser adapt for oceanbase 4.3.3 (#3762) * adapt for oceanbase mysql * make ob-oracle adapt for ob433 * add more test cases * rename class * fix array out of bound err --- client | 2 +- .../mysql/MySQLAlterTableActionFactory.java | 51 ++- .../adapter/mysql/MySQLAlterTableFactory.java | 15 +- .../mysql/MySQLCreateTableFactory.java | 8 +- .../adapter/mysql/MySQLDataTypeFactory.java | 2 +- .../adapter/mysql/MySQLExpressionFactory.java | 255 ++++++++--- .../mysql/MySQLFromReferenceFactory.java | 30 +- .../mysql/MySQLIndexOptionsFactory.java | 9 + .../adapter/mysql/MySQLInsertFactory.java | 10 +- .../adapter/mysql/MySQLSelectBodyFactory.java | 9 +- .../mysql/MySQLTableElementFactory.java | 26 +- .../mysql/MySQLTableOptionsFactory.java | 70 +++ .../oracle/OracleAlterTableActionFactory.java | 45 +- .../oracle/OracleAlterTableFactory.java | 5 +- .../oracle/OracleCreateIndexFactory.java | 3 + .../adapter/oracle/OracleDataTypeFactory.java | 2 + .../oracle/OracleExpressionFactory.java | 427 ++++++++++++------ .../oracle/OracleFromReferenceFactory.java | 6 +- .../adapter/oracle/OracleInsertFactory.java | 12 +- .../oracle/OraclePartitionUsageFactory.java | 26 +- .../oracle/OracleTableElementFactory.java | 4 + .../oracle/OracleTableOptionsFactory.java | 20 + .../alter/table/AlterTableAction.java | 32 +- .../statement/common/mysql/ArrayType.java | 12 +- .../common/mysql/LobStorageOption.java | 61 +++ .../statement/common/mysql/VectorType.java | 6 +- .../statement/createindex/CreateIndex.java | 4 + .../createtable/ColumnAttributes.java | 12 + .../createtable/ColumnDefinition.java | 10 + .../statement/createtable/CreateTable.java | 8 + .../statement/createtable/IndexOptions.java | 10 + .../statement/createtable/OutOfLineIndex.java | 2 + .../statement/createtable/TableOptions.java | 87 ++++ .../statement/expression/ArrayExpression.java | 5 +- .../statement/expression/FullTextSearch.java | 5 + .../{JsonConstraint.java => JsonOption.java} | 38 +- .../sqlparser/statement/insert/Insert.java | 14 + .../statement/select/ExpressionReference.java | 8 +- .../statement/select/PartitionUsage.java | 26 +- .../statement/select/SelectBody.java | 11 +- .../main/resources/obmysql/sql/OBParser.g4 | 87 +++- .../main/resources/oboracle/sql/OBParser.g4 | 44 +- .../MySQLAlterTableActionFactoryTest.java | 42 ++ .../adapter/MySQLAlterTableFactoryTest.java | 51 +++ .../adapter/MySQLCreateIndexFactoryTest.java | 17 +- .../adapter/MySQLCreateTableFactoryTest.java | 83 +++- .../adapter/MySQLExpressionFactoryTest.java | 372 +++++++++++++-- .../MySQLFromReferenceFactoryTest.java | 42 +- .../adapter/MySQLInsertFactoryTest.java | 18 + .../adapter/MySQLSelectFactoryTest.java | 7 +- .../adapter/MySQLTableElementFactoryTest.java | 65 ++- .../OracleAlterTableActionFactoryTest.java | 11 + .../adapter/OracleAlterTableFactoryTest.java | 51 +++ .../adapter/OracleCreateIndexFactoryTest.java | 14 + .../adapter/OracleCreateTableFactoryTest.java | 17 +- .../adapter/OracleExpressionFactoryTest.java | 272 +++++++---- .../adapter/OracleInsertFactoryTest.java | 43 +- 57 files changed, 2154 insertions(+), 470 deletions(-) create mode 100644 libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/LobStorageOption.java rename libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/{JsonConstraint.java => JsonOption.java} (72%) diff --git a/client b/client index 39844a21e4..dba42e3ab0 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 39844a21e4d76aa1f70ea67c981c74ac56ba38fb +Subproject commit dba42e3ab0e4fe9edbc7de3dc0d3495bfbfb6a0f diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableActionFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableActionFactory.java index 96502b8838..6526049e79 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableActionFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableActionFactory.java @@ -15,18 +15,18 @@ */ package com.oceanbase.tools.sqlparser.adapter.mysql; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Add_external_table_partition_actionsContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_behaviorContext; -import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_group_optionContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_group_actionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_optionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_constraint_optionContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_external_table_actionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_index_optionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_partition_optionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_table_actionContext; @@ -35,6 +35,7 @@ import com.oceanbase.tools.sqlparser.obmysql.OBParser.Name_listContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Opt_partition_range_or_listContext; import com.oceanbase.tools.sqlparser.obmysql.OBParserBaseVisitor; +import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTableAction; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTableAction.AlterColumnBehavior; import com.oceanbase.tools.sqlparser.statement.common.ColumnGroupElement; @@ -63,10 +64,14 @@ public MySQLAlterTableActionFactory(@NonNull Alter_table_actionContext alterTabl this.parserRuleContext = alterTableActionContext; } - public MySQLAlterTableActionFactory(@NonNull Alter_column_group_optionContext alterTableActionContext) { + public MySQLAlterTableActionFactory(@NonNull Alter_column_group_actionContext alterTableActionContext) { this.parserRuleContext = alterTableActionContext; } + public MySQLAlterTableActionFactory(@NonNull Alter_external_table_actionContext alterExternalTableActionContext) { + this.parserRuleContext = alterExternalTableActionContext; + } + @Override public AlterTableAction generate() { return visit(this.parserRuleContext); @@ -95,6 +100,20 @@ public AlterTableAction visitAlter_table_action(Alter_table_actionContext ctx) { return visitChildren(ctx); } + @Override + public AlterTableAction visitAlter_external_table_action(Alter_external_table_actionContext ctx) { + AlterTableAction action = new AlterTableAction(ctx); + action.setExternalTableLocation(ctx.STRING_VALUE().getText()); + if (ctx.DROP() != null && ctx.PARTITION() != null) { + action.setDropExternalTablePartition(true); + } else if (ctx.ADD() != null && ctx.PARTITION() != null) { + Map externalTablePartition = new HashMap<>(); + visitAddExternalTablePartitionActions(externalTablePartition, ctx.add_external_table_partition_actions()); + action.setAddExternalTablePartition(externalTablePartition); + } + return action; + } + @Override public AlterTableAction visitAlter_column_option(Alter_column_optionContext ctx) { AlterTableAction alterTableAction = new AlterTableAction(ctx); @@ -108,6 +127,10 @@ public AlterTableAction visitAlter_column_option(Alter_column_optionContext ctx) .collect(Collectors.toList()); } alterTableAction.setAddColumns(addColumns); + if (ctx.lob_storage_clause() != null) { + alterTableAction.setLobStorageOption( + MySQLTableOptionsFactory.getLobStorageOption(ctx.lob_storage_clause())); + } } else if (ctx.DROP() != null) { String option = null; if (ctx.CASCADE() != null) { @@ -131,6 +154,8 @@ public AlterTableAction visitAlter_column_option(Alter_column_optionContext ctx) AlterColumnBehavior behavior = new AlterColumnBehavior(aCtx); if (aCtx.signed_literal() != null) { behavior.setDefaultValue(MySQLTableElementFactory.getSignedLiteral(aCtx.signed_literal())); + } else if (aCtx.expr() != null) { + behavior.setDefaultValue(new MySQLExpressionFactory(aCtx.expr()).generate()); } alterTableAction.alterColumnBehavior(colRef, behavior); } else if (ctx.RENAME() != null) { @@ -211,6 +236,9 @@ public AlterTableAction visitAlter_partition_option(Alter_partition_optionContex getPartitionElements(ctx.opt_partition_range_or_list())); } else if (ctx.REMOVE() != null && ctx.PARTITIONING() != null) { alterTableAction.setRemovePartitioning(true); + } else if (ctx.EXCHANGE() != null && ctx.PARTITION() != null) { + alterTableAction.setExchangePartition(ctx.relation_name().getText(), + MySQLFromReferenceFactory.getRelationFactor(ctx.relation_factor())); } return alterTableAction; } @@ -241,7 +269,7 @@ public AlterTableAction visitAlter_constraint_option(Alter_constraint_optionCont } @Override - public AlterTableAction visitAlter_column_group_option(Alter_column_group_optionContext ctx) { + public AlterTableAction visitAlter_column_group_action(Alter_column_group_actionContext ctx) { AlterTableAction action = new AlterTableAction(ctx); List columnGroupElements = ctx.column_group_list().column_group_element() .stream().map(c -> new MySQLColumnGroupElementFactory(c).generate()).collect(Collectors.toList()); @@ -273,4 +301,15 @@ private List getPartitionElements(Opt_partition_range_or_listC .map(c -> new MySQLPartitionElementFactory(c).generate()).collect(Collectors.toList()); } + private void visitAddExternalTablePartitionActions(Map externalTablePartition, + Add_external_table_partition_actionsContext context) { + if (context == null) { + return; + } + Expression value = new MySQLExpressionFactory() + .visit(context.add_external_table_partition_action().expr_const()); + externalTablePartition.put(context.add_external_table_partition_action().column_name().getText(), value); + visitAddExternalTablePartitionActions(externalTablePartition, context.add_external_table_partition_actions()); + } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableFactory.java index 415eede3e2..1f0d6cd107 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableFactory.java @@ -22,7 +22,8 @@ import org.antlr.v4.runtime.ParserRuleContext; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; -import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_group_optionContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_group_actionContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_external_table_actionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_table_actionsContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_table_stmtContext; import com.oceanbase.tools.sqlparser.obmysql.OBParserBaseVisitor; @@ -58,8 +59,10 @@ public AlterTable visitAlter_table_stmt(Alter_table_stmtContext ctx) { List actions = null; if (ctx.alter_table_actions() != null) { actions = getAlterTableActions(ctx.alter_table_actions()); - } else if (ctx.alter_column_group_option() != null) { - actions = getAlterTableActions(ctx.alter_column_group_option()); + } else if (ctx.alter_column_group_action() != null) { + actions = getAlterTableActions(ctx.alter_column_group_action()); + } else if (ctx.alter_external_table_action() != null) { + actions = getAlterTableActions(ctx.alter_external_table_action()); } AlterTable alterTable = new AlterTable(ctx, factor, actions); if (ctx.EXTERNAL() != null) { @@ -83,7 +86,11 @@ private List getAlterTableActions(Alter_table_actionsContext c return actions; } - private List getAlterTableActions(Alter_column_group_optionContext context) { + private List getAlterTableActions(Alter_column_group_actionContext context) { + return Collections.singletonList(new MySQLAlterTableActionFactory(context).generate()); + } + + private List getAlterTableActions(Alter_external_table_actionContext context) { return Collections.singletonList(new MySQLAlterTableActionFactory(context).generate()); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLCreateTableFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLCreateTableFactory.java index 35bb5c5e52..4f23429d73 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLCreateTableFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLCreateTableFactory.java @@ -69,8 +69,7 @@ public CreateTable visitCreate_table_like_stmt(Create_table_like_stmtContext ctx createTable.setIfNotExists(true); } createTable.setUserVariable(factor.getUserVariable()); - RelationFactor likeFactor = MySQLFromReferenceFactory.getRelationFactor(ctx.relation_factor(1)); - createTable.setLikeTable(likeFactor); + createTable.setLikeTable(MySQLFromReferenceFactory.getRelationFactor(ctx.relation_factor(1))); return createTable; } @@ -86,7 +85,6 @@ public CreateTable visitCreate_table_stmt(Create_table_stmtContext ctx) { createTable.setTemporary(true); } } - if (ctx.IF() != null && ctx.not() != null && ctx.EXISTS() != null) { createTable.setIfNotExists(true); } @@ -112,6 +110,10 @@ public CreateTable visitCreate_table_stmt(Create_table_stmtContext ctx) { .map(c -> new MySQLColumnGroupElementFactory(c).generate()).collect(Collectors.toList()); createTable.setColumnGroupElements(columnGroupElements); } + if (ctx.ignore_or_replace() != null) { + createTable.setIgnore(ctx.ignore_or_replace().IGNORE() != null); + createTable.setReplace(ctx.ignore_or_replace().REPLACE() != null); + } return createTable; } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLDataTypeFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLDataTypeFactory.java index f46fb7cb16..8ece4121bd 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLDataTypeFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLDataTypeFactory.java @@ -90,7 +90,7 @@ public DataType visitData_type(Data_typeContext ctx) { if (ctx.STRING_VALUE() != null) { return new GeneralDataType(ctx, ctx.STRING_VALUE().getText(), null); } else if (ctx.data_type() != null) { - return new ArrayType(visitData_type(ctx.data_type())); + return new ArrayType(ctx, visitData_type(ctx.data_type())); } return visitChildren(ctx); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLExpressionFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLExpressionFactory.java index 0f072f42ea..d9d1d60b1b 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLExpressionFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLExpressionFactory.java @@ -45,6 +45,7 @@ import com.oceanbase.tools.sqlparser.obmysql.OBParser.Expr_listContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.In_exprContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Json_on_responseContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Json_query_exprContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Json_table_column_defContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Json_table_exists_column_defContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Json_table_exprContext; @@ -56,7 +57,11 @@ import com.oceanbase.tools.sqlparser.obmysql.OBParser.Mock_jt_on_error_on_emptyContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Mvt_paramContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.On_emptyContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.On_empty_queryContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.On_errorContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.On_error_queryContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.On_mismatch_queryContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Opt_response_queryContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Opt_value_on_empty_or_error_or_mismatchContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Parameterized_trimContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.PredicateContext; @@ -68,8 +73,10 @@ import com.oceanbase.tools.sqlparser.obmysql.OBParser.Ttl_exprContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Utc_time_funcContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Utc_timestamp_funcContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Vector_distance_exprContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Win_fun_first_last_paramsContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Window_functionContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Wrapper_optsContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Ws_nweightsContext; import com.oceanbase.tools.sqlparser.obmysql.OBParserBaseVisitor; import com.oceanbase.tools.sqlparser.statement.Expression; @@ -79,24 +86,7 @@ import com.oceanbase.tools.sqlparser.statement.common.CharacterType; import com.oceanbase.tools.sqlparser.statement.common.GeneralDataType; import com.oceanbase.tools.sqlparser.statement.common.WindowSpec; -import com.oceanbase.tools.sqlparser.statement.expression.ArrayExpression; -import com.oceanbase.tools.sqlparser.statement.expression.BoolValue; -import com.oceanbase.tools.sqlparser.statement.expression.CaseWhen; -import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; -import com.oceanbase.tools.sqlparser.statement.expression.CompoundExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; -import com.oceanbase.tools.sqlparser.statement.expression.DefaultExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ExpressionParam; -import com.oceanbase.tools.sqlparser.statement.expression.FullTextSearch; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionParam; -import com.oceanbase.tools.sqlparser.statement.expression.GroupConcat; -import com.oceanbase.tools.sqlparser.statement.expression.IntervalExpression; -import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption; -import com.oceanbase.tools.sqlparser.statement.expression.NullExpression; -import com.oceanbase.tools.sqlparser.statement.expression.TextSearchMode; -import com.oceanbase.tools.sqlparser.statement.expression.WhenClause; +import com.oceanbase.tools.sqlparser.statement.expression.*; import lombok.NonNull; @@ -223,6 +213,9 @@ public Expression visitBool_pri(Bool_priContext ctx) { } else if (ctx.bool_pri() != null && ctx.any_expr() != null) { left = visit(ctx.bool_pri()); right = visit(ctx.any_expr()); + } else if (ctx.bool_pri() != null && ctx.select_with_parens() != null) { + left = visit(ctx.bool_pri()); + right = new MySQLSelectBodyFactory(ctx.select_with_parens()).generate(); } if (left == null || right == null || operator == null) { throw new IllegalStateException("Unable to build expression, some syntax modules are missing"); @@ -389,6 +382,9 @@ public Expression visitSimple_expr(Simple_exprContext ctx) { } FullTextSearch f = new FullTextSearch(ctx, params, ctx.STRING_VALUE().getText()); f.setSearchMode(searchMode); + if (ctx.WITH() != null && ctx.QUERY() != null && ctx.EXPANSION() != null) { + f.setWithQueryExpansion(true); + } return f; } else if (ctx.func_expr() != null) { return visit(ctx.func_expr()); @@ -477,6 +473,8 @@ public Expression visitSimple_func_expr(Simple_func_exprContext ctx) { p.addOption(new ConstExpression(e.STRING_VALUE())); return p; }).collect(Collectors.toList())); + } else if (ctx.INTNUM() != null) { + params.add(new ExpressionParam(new ConstExpression(ctx.INTNUM()))); } else if (ctx.column_ref() != null) { params.add(new ExpressionParam(new MySQLColumnRefFactory(ctx.column_ref()).generate())); if (CollectionUtils.isNotEmpty(ctx.mvt_param())) { @@ -533,6 +531,9 @@ public Expression visitComplex_func_expr(Complex_func_exprContext ctx) { } else if (ctx.CAST() != null) { FunctionParam p = wrap(ctx.expr()); p.addOption(new MySQLDataTypeFactory(ctx.cast_data_type()).generate()); + if (ctx.ARRAY() != null) { + p.addOption(new ConstExpression(ctx.ARRAY())); + } return new FunctionCall(ctx, ctx.CAST().getText(), Collections.singletonList(p)); } else if (ctx.CONVERT() != null) { FunctionParam p = wrap(ctx.expr()); @@ -635,36 +636,11 @@ public Expression visitComplex_func_expr(Complex_func_exprContext ctx) { } return fCall; } else if (ctx.json_value_expr() != null) { - Json_value_exprContext jsonValue = ctx.json_value_expr(); - List params = new ArrayList<>(); - params.add(new ExpressionParam(visit(jsonValue.simple_expr()))); - params.add(new ExpressionParam(visit(jsonValue.complex_string_literal()))); - FunctionCall fCall = new FunctionCall(ctx, jsonValue.JSON_VALUE().getText(), params); - if (jsonValue.cast_data_type() != null) { - fCall.addOption(new MySQLDataTypeFactory(jsonValue.cast_data_type()).generate()); - } - if (jsonValue.TRUNCATE() != null) { - fCall.addOption(new ConstExpression(jsonValue.TRUNCATE())); - } - if (jsonValue.ASCII() != null) { - fCall.addOption(new ConstExpression(jsonValue.ASCII())); - } - JsonOnOption jsonOnOption; - if (jsonValue.on_empty() != null && jsonValue.on_error() != null) { - jsonOnOption = new JsonOnOption(jsonValue.on_empty(), jsonValue.on_error()); - setOnError(jsonOnOption, jsonValue.on_error()); - setOnEmpty(jsonOnOption, jsonValue.on_empty()); - fCall.addOption(jsonOnOption); - } else if (jsonValue.on_error() != null) { - jsonOnOption = new JsonOnOption(jsonValue.on_error()); - setOnError(jsonOnOption, jsonValue.on_error()); - fCall.addOption(jsonOnOption); - } else if (jsonValue.on_empty() != null) { - jsonOnOption = new JsonOnOption(jsonValue.on_empty()); - setOnEmpty(jsonOnOption, jsonValue.on_empty()); - fCall.addOption(jsonOnOption); - } - return fCall; + return visit(ctx.json_value_expr()); + } else if (ctx.json_query_expr() != null) { + return visit(ctx.json_query_expr()); + } else if (ctx.vector_distance_expr() != null) { + return visit(ctx.vector_distance_expr()); } String funcName = null; List params = new ArrayList<>(); @@ -711,10 +687,6 @@ public Expression visitComplex_func_expr(Complex_func_exprContext ctx) { } else if (ctx.sys_interval_func().CHECK() != null) { funcName = ctx.sys_interval_func().CHECK().getText(); } - } else if (ctx.vector_distance_expr() != null) { - params = ctx.vector_distance_expr().expr() - .stream().map(this::wrap).collect(Collectors.toList()); - funcName = ctx.vector_distance_expr().VECTOR_DISTANCE().getText(); } if (funcName == null) { throw new IllegalStateException("Missing function name"); @@ -722,11 +694,93 @@ public Expression visitComplex_func_expr(Complex_func_exprContext ctx) { return new FunctionCall(ctx, funcName, params); } + @Override + public Expression visitJson_query_expr(Json_query_exprContext ctx) { + List params = new ArrayList<>(); + params.add(new ExpressionParam(visit(ctx.simple_expr()))); + params.add(new ExpressionParam(visit(ctx.complex_string_literal()))); + FunctionCall fCall = new FunctionCall(ctx, ctx.JSON_QUERY().getText(), params); + if (ctx.cast_data_type() != null) { + fCall.addOption(new MySQLDataTypeFactory(ctx.cast_data_type()).generate()); + } + if (ctx.json_query_opt() != null) { + JsonOption jsonOpt = new JsonOption(ctx.json_query_opt()); + fCall.addOption(jsonOpt); + if (ctx.json_query_opt().TRUNCATE() != null) { + jsonOpt.setTruncate(true); + } + if (ctx.json_query_opt().scalars_opt() != null) { + if (ctx.json_query_opt().scalars_opt().ALLOW() != null) { + jsonOpt.setScalarsMode(JsonOption.ScalarsMode.ALLOW_SCALARS); + } else { + jsonOpt.setScalarsMode(JsonOption.ScalarsMode.DISALLOW_SCALARS); + } + } + if (ctx.json_query_opt().PRETTY() != null) { + jsonOpt.setPretty(true); + } + if (ctx.json_query_opt().ASCII() != null) { + jsonOpt.setAscii(true); + } + setWrapperMode(jsonOpt, ctx.json_query_opt().wrapper_opts()); + if (ctx.json_query_opt().ASIS() != null) { + jsonOpt.setAsis(true); + } + if (ctx.json_query_opt().json_query_on_opt() != null) { + JsonOnOption jsonOnOption = new JsonOnOption(ctx.json_query_opt().json_query_on_opt()); + jsonOpt.setOnOption(jsonOnOption); + setOnError(jsonOnOption, ctx.json_query_opt().json_query_on_opt().on_error_query()); + setOnEmpty(jsonOnOption, ctx.json_query_opt().json_query_on_opt().on_empty_query()); + setOnMismatch(jsonOnOption, ctx.json_query_opt().json_query_on_opt().on_mismatch_query()); + } + if (ctx.json_query_opt().MULTIVALUE() != null) { + jsonOpt.setMultiValue(true); + } + } + return fCall; + } + + @Override + public Expression visitVector_distance_expr(Vector_distance_exprContext ctx) { + List params = ctx.expr().stream().map(this::wrap).collect(Collectors.toList()); + if (ctx.vector_distance_metric() != null) { + params.add(new ExpressionParam(new ConstExpression(ctx.vector_distance_metric()))); + } + return new FunctionCall(ctx, ctx.VECTOR_DISTANCE().getText(), params); + } + + @Override + public Expression visitJson_value_expr(Json_value_exprContext ctx) { + List params = new ArrayList<>(); + params.add(new ExpressionParam(visit(ctx.simple_expr()))); + params.add(new ExpressionParam(visit(ctx.complex_string_literal()))); + FunctionCall fCall = new FunctionCall(ctx, ctx.JSON_VALUE().getText(), params); + if (ctx.cast_data_type() != null) { + fCall.addOption(new MySQLDataTypeFactory(ctx.cast_data_type()).generate()); + } + if (ctx.json_value_opt() != null) { + JsonOption jsonOpt = new JsonOption(ctx.json_value_opt()); + fCall.addOption(jsonOpt); + if (ctx.json_value_opt().TRUNCATE() != null) { + jsonOpt.setTruncate(true); + } + if (ctx.json_value_opt().ASCII() != null) { + jsonOpt.setAscii(true); + } + if (ctx.json_value_opt().json_value_on_opt() != null) { + JsonOnOption jsonOnOption = new JsonOnOption(ctx.json_value_opt().json_value_on_opt()); + jsonOpt.setOnOption(jsonOnOption); + setOnError(jsonOnOption, ctx.json_value_opt().json_value_on_opt().on_error()); + setOnEmpty(jsonOnOption, ctx.json_value_opt().json_value_on_opt().on_empty()); + } + } + return fCall; + } + @Override public Expression visitTtl_expr(Ttl_exprContext ctx) { Expression right = new IntervalExpression(ctx.INTERVAL(), ctx.ttl_unit(), new ConstExpression(ctx.INTNUM()), ctx.ttl_unit().getText()); - return new CompoundExpression(ctx, new MySQLColumnRefFactory(ctx.column_definition_ref()).generate(), right, Operator.ADD); } @@ -881,7 +935,7 @@ public Expression visitJson_table_expr(Json_table_exprContext ctx) { FunctionCall fCall = new FunctionCall(ctx, ctx.JSON_TABLE().getText(), Arrays.asList(new ExpressionParam(visit(ctx.simple_expr())), new ExpressionParam(visit(ctx.literal())))); - fCall.addOption(getJsonOnOption(ctx.mock_jt_on_error_on_empty())); + fCall.addOption(getJsonOption(ctx.mock_jt_on_error_on_empty())); ctx.jt_column_list().json_table_column_def().forEach(c -> fCall.addOption(visitJsonTableColumnDef(c))); return fCall; } @@ -894,7 +948,12 @@ public Expression visitAny_expr(Any_exprContext ctx) { return visit(ctx.expr_list()); } - private JsonOnOption getJsonOnOption(Mock_jt_on_error_on_emptyContext ctx) { + @Override + public Expression visitOpt_response_query(Opt_response_queryContext ctx) { + return ctx.NULLX() != null ? new ConstExpression(ctx.NULLX()) : new ConstExpression(ctx.ERROR_P()); + } + + private JsonOnOption getJsonOption(Mock_jt_on_error_on_emptyContext ctx) { return null; } @@ -924,6 +983,53 @@ private void setOnError(JsonOnOption jsonOnOption, On_errorContext ctx) { jsonOnOption.setOnError(visit(ctx.json_on_response())); } + private void setOnEmpty(JsonOnOption jsonOnOption, On_empty_queryContext ctx) { + if (ctx == null) { + return; + } + if (ctx.opt_response_query() != null) { + jsonOnOption.setOnEmpty(visit(ctx.opt_response_query())); + } else if (ctx.EMPTY().size() == 2) { + TerminalNode endNode = ctx.EMPTY(0); + if (ctx.ARRAY() != null) { + endNode = ctx.ARRAY(); + } else if (ctx.OBJECT() != null) { + endNode = ctx.OBJECT(); + } + jsonOnOption.setOnEmpty(new ConstExpression(ctx.EMPTY(0), endNode)); + } + } + + private void setOnError(JsonOnOption jsonOnOption, On_error_queryContext ctx) { + if (ctx == null) { + return; + } + if (ctx.opt_response_query() != null) { + jsonOnOption.setOnError(visit(ctx.opt_response_query())); + } else if (ctx.EMPTY() != null) { + TerminalNode endNode = ctx.EMPTY(); + if (ctx.ARRAY() != null) { + endNode = ctx.ARRAY(); + } else if (ctx.OBJECT() != null) { + endNode = ctx.OBJECT(); + } + jsonOnOption.setOnError(new ConstExpression(ctx.EMPTY(), endNode)); + } + } + + private void setOnMismatch(JsonOnOption jsonOnOption, On_mismatch_queryContext ctx) { + if (ctx == null) { + return; + } + Expression opt; + if (ctx.DOT() != null) { + opt = new ConstExpression(ctx.DOT()); + } else { + opt = visit(ctx.opt_response_query()); + } + jsonOnOption.setOnMismatches(Collections.singletonList(new JsonOnOption.OnMismatch(ctx, opt, null))); + } + private FunctionParam visitJsonTableColumnDef(Json_table_column_defContext ctx) { if (ctx.json_table_ordinality_column_def() != null) { return visitJsonTableOrdinalityColumnDef(ctx.json_table_ordinality_column_def()); @@ -951,7 +1057,7 @@ private FunctionParam visitJsonTableExistsColumnDef(Json_table_exists_column_def } param.addOption(new ConstExpression(ctx.EXISTS())); param.addOption(visit(ctx.literal())); - param.addOption(getJsonOnOption(ctx.mock_jt_on_error_on_empty())); + param.addOption(getJsonOption(ctx.mock_jt_on_error_on_empty())); return param; } @@ -963,7 +1069,11 @@ private FunctionParam visitJsonTableValueColumnDef(Json_table_value_column_defCo param.addOption(new ConstExpression(ctx.collation().collation_name())); } param.addOption(visit(ctx.literal())); - param.addOption(getJsonOnOption(ctx.opt_value_on_empty_or_error_or_mismatch())); + if (ctx.opt_value_on_empty_or_error_or_mismatch() != null) { + JsonOption jsonOpt = new JsonOption(ctx.opt_value_on_empty_or_error_or_mismatch()); + param.addOption(jsonOpt); + jsonOpt.setOnOption(getJsonOnOption(ctx.opt_value_on_empty_or_error_or_mismatch())); + } return param; } @@ -1004,4 +1114,35 @@ private ConstExpression getAggregator(Complex_func_exprContext ctx) { return null; } + private void setWrapperMode(JsonOption c, Wrapper_optsContext ctx) { + if (ctx == null) { + return; + } + if (ctx.WITH() != null) { + if (ctx.ARRAY() != null) { + if (ctx.CONDITIONAL() != null) { + c.setWrapperMode(JsonOption.WrapperMode.WITH_CONDITIONAL_ARRAY_WRAPPER); + } else if (ctx.UNCONDITIONAL() != null) { + c.setWrapperMode(JsonOption.WrapperMode.WITH_UNCONDITIONAL_ARRAY_WRAPPER); + } else { + c.setWrapperMode(JsonOption.WrapperMode.WITH_ARRAY_WRAPPER); + } + } else { + if (ctx.CONDITIONAL() != null) { + c.setWrapperMode(JsonOption.WrapperMode.WITH_CONDITIONAL_WRAPPER); + } else if (ctx.UNCONDITIONAL() != null) { + c.setWrapperMode(JsonOption.WrapperMode.WITH_UNCONDITIONAL_WRAPPER); + } else { + c.setWrapperMode(JsonOption.WrapperMode.WITH_WRAPPER); + } + } + } else { + if (ctx.ARRAY() != null) { + c.setWrapperMode(JsonOption.WrapperMode.WITHOUT_ARRAY_WRAPPER); + } else { + c.setWrapperMode(JsonOption.WrapperMode.WITHOUT_WRAPPER); + } + } + } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLFromReferenceFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLFromReferenceFactory.java index 75b330b7ca..2701e02480 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLFromReferenceFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLFromReferenceFactory.java @@ -15,9 +15,7 @@ */ package com.oceanbase.tools.sqlparser.adapter.mysql; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; @@ -108,7 +106,11 @@ public FromReference visitTable_factor(Table_factorContext ctx) { if (ctx.tbl_name() != null) { return visit(ctx.tbl_name()); } else if (ctx.table_subquery() != null) { - return visit(ctx.table_subquery()); + ExpressionReference reference = (ExpressionReference) visit(ctx.table_subquery()); + if (ctx.LATERAL() != null) { + reference.setLateral(true); + } + return reference; } else if (ctx.table_reference() != null) { FromReference from = visit(ctx.table_reference()); if (ctx.LeftBrace() == null || ctx.OJ() == null || ctx.RightBrace() == null) { @@ -123,6 +125,9 @@ public FromReference visitTable_factor(Table_factorContext ctx) { } StatementFactory factory = new MySQLSelectBodyFactory(ctx.select_with_parens()); ExpressionReference reference = new ExpressionReference(ctx, factory.generate(), null); + if (ctx.LATERAL() != null) { + reference.setLateral(true); + } if (ctx.use_flashback() != null) { reference.setFlashbackUsage(visitFlashbackUsage(ctx.use_flashback())); } @@ -193,7 +198,7 @@ public FromReference visitTbl_name(Tbl_nameContext ctx) { } NameReference nameReference = new NameReference(ctx, factor.getSchema(), factor.getRelation(), alias); if (ctx.use_partition() != null) { - nameReference.setPartitionUsage(visitPartitonUsage(ctx.use_partition())); + nameReference.setPartitionUsage(visitPartitionUsage(ctx.use_partition())); } if (ctx.use_flashback() != null) { nameReference.setFlashbackUsage(visitFlashbackUsage(ctx.use_flashback())); @@ -296,10 +301,17 @@ private FlashbackUsage visitFlashbackUsage(Use_flashbackContext ctx) { return new FlashbackUsage(ctx, FlashBackType.AS_OF_SNAPSHOT, factory.generate()); } - public static PartitionUsage visitPartitonUsage(Use_partitionContext usePartition) { - List nameList = new ArrayList<>(); - visitNameList(usePartition.name_list(), nameList); - return new PartitionUsage(usePartition, PartitionType.PARTITION, nameList); + public static PartitionUsage visitPartitionUsage(Use_partitionContext usePartition) { + if (usePartition.name_list() != null) { + List nameList = new ArrayList<>(); + visitNameList(usePartition.name_list(), nameList); + return new PartitionUsage(usePartition, PartitionType.PARTITION, nameList); + } + Map externalTablePartition = new HashMap<>(); + usePartition.external_table_partitions().external_table_partition() + .forEach(ctx -> externalTablePartition.put(ctx.relation_name().getText(), + new MySQLExpressionFactory().visit(ctx.expr_const()))); + return new PartitionUsage(usePartition, PartitionType.PARTITION, externalTablePartition); } private static void visitNameList(Name_listContext ctx, List nameList) { diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java index bcc5217006..2888e4a780 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java @@ -15,7 +15,9 @@ */ package com.oceanbase.tools.sqlparser.adapter.mysql; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; @@ -61,6 +63,8 @@ public IndexOptions visitOpt_index_options(Opt_index_optionsContext ctx) { indexOptions.setGlobal(false); } else if (option.BLOCK_SIZE() != null) { indexOptions.setBlockSize(getInteger(option)); + } else if (option.KEY_BLOCK_SIZE() != null) { + indexOptions.setKeyBlockSize(getInteger(option)); } else if (option.DATA_TABLE_ID() != null) { indexOptions.setDataTableId(getInteger(option)); } else if (option.INDEX_TABLE_ID() != null) { @@ -79,6 +83,11 @@ public IndexOptions visitOpt_index_options(Opt_index_optionsContext ctx) { indexOptions.setWithParser(option.STRING_VALUE().getText()); } else if (option.WITH() != null && option.ROWID() != null) { indexOptions.setWithRowId(true); + } else if (option.WITH() != null && option.vec_index_params() != null) { + Map params = new HashMap<>(); + option.vec_index_params().vec_index_param() + .forEach(i -> params.put(i.relation_name().getText(), i.vec_index_param_value().getText())); + indexOptions.setVectorIndexParams(params); } else if (option.index_using_algorithm() != null) { indexOptions.merge(visit(option.index_using_algorithm())); } else if (option.visibility_option() != null) { diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLInsertFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLInsertFactory.java index 597e97f07d..4c5b491fe4 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLInsertFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLInsertFactory.java @@ -67,6 +67,14 @@ public Insert visitInsert_stmt(Insert_stmtContext ctx) { if (ctx.IGNORE() != null) { insert.setIgnore(true); } + if (ctx.HIGH_PRIORITY() != null) { + insert.setHighPriority(true); + } else if (ctx.LOW_PRIORITY() != null) { + insert.setLowPriority(true); + } + if (ctx.OVERWRITE() != null) { + insert.setOverwrite(true); + } if (ctx.update_asgn_list() != null) { insert.setOnDuplicateKeyUpdateColumns(getSetColumns(ctx.update_asgn_list())); } @@ -79,7 +87,7 @@ public Insert visitSingle_table_insert(Single_table_insertContext ctx) { .getRelationFactor(ctx.dml_table_name().relation_factor())); if (ctx.dml_table_name().use_partition() != null) { insertTable.setPartitionUsage(MySQLFromReferenceFactory - .visitPartitonUsage(ctx.dml_table_name().use_partition())); + .visitPartitionUsage(ctx.dml_table_name().use_partition())); } if (ctx.column_list() != null) { insertTable.setColumns(ctx.column_list().column_definition_ref().stream() diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLSelectBodyFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLSelectBodyFactory.java index 9b4f351fe0..865ad6c677 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLSelectBodyFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLSelectBodyFactory.java @@ -231,12 +231,13 @@ private RelationType getRelationType(Set_typeContext setType) { public SelectBody visitSimple_select_with_order_and_limit(Simple_select_with_order_and_limitContext ctx) { SelectBody select = new SelectBody(ctx, visit(ctx.simple_select())); if (ctx.order_by() != null) { - StatementFactory factory = new MySQLOrderByFactory(ctx.order_by()); - select.setOrderBy(factory.generate()); + select.setOrderBy(new MySQLOrderByFactory(ctx.order_by()).generate()); + } + if (ctx.opt_approx() != null) { + select.setApproximate(true); } if (ctx.limit_clause() != null) { - StatementFactory factory = new MySQLLimitFactory(ctx.limit_clause()); - select.setLimit(factory.generate()); + select.setLimit(new MySQLLimitFactory(ctx.limit_clause()).generate()); } return select; } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableElementFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableElementFactory.java index 17f63470df..3460a75e3f 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableElementFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableElementFactory.java @@ -243,6 +243,12 @@ public TableElement visitColumn_definition(Column_definitionContext ctx) { ColumnAttributes attributes = visitGeneratedColumnAttributeList(ctx.opt_generated_column_attribute_list()); definition.setColumnAttributes(attributes); } + if (ctx.references_clause() != null) { + definition.setForeignReference(visitForeignReference(ctx.references_clause())); + } + if (ctx.SERIAL() != null) { + definition.setSerial(true); + } if (ctx.FIRST() != null) { definition.setLocation(new Location(ctx.FIRST().getText(), null)); } else if (ctx.BEFORE() != null) { @@ -422,8 +428,14 @@ private ColumnAttributes visitColumnAttribute(Column_attributeContext ctx) { attribute = new InLineCheckConstraint(ctx, constraintName, state, new MySQLExpressionFactory(ctx.expr()).generate()); attributes.setConstraints(Collections.singletonList(attribute)); - } else if (ctx.DEFAULT() != null || ctx.ORIG_DEFAULT() != null) { - Expression expr = visitNowOrSignedLiteral(ctx.now_or_signed_literal()); + } else if ((ctx.DEFAULT() != null || ctx.ORIG_DEFAULT() != null) + && (ctx.now_or_signed_literal() != null || ctx.expr() != null)) { + Expression expr = null; + if (ctx.now_or_signed_literal() != null) { + expr = visitNowOrSignedLiteral(ctx.now_or_signed_literal()); + } else if (ctx.expr() != null) { + expr = new MySQLExpressionFactory(ctx.expr()).generate(); + } if (ctx.DEFAULT() != null) { attributes.setDefaultValue(expr); } else { @@ -450,6 +462,16 @@ private ColumnAttributes visitColumnAttribute(Column_attributeContext ctx) { skipIndexTypes.add(ctx.skip_index_type().getText()); } attributes.setSkipIndexTypes(skipIndexTypes); + } else if (ctx.lob_chunk_size() != null) { + if (ctx.lob_chunk_size().STRING_VALUE() != null) { + attributes.setLobChunkSize(ctx.lob_chunk_size().STRING_VALUE().getText()); + } else if (ctx.lob_chunk_size().INTNUM() != null) { + attributes.setLobChunkSize(ctx.lob_chunk_size().INTNUM().getText()); + } + } else if (ctx.COLUMN_FORMAT() != null) { + attributes.setColumnFormat(ctx.col_attri_value.getText()); + } else if (ctx.STORAGE() != null) { + attributes.setStorage(ctx.col_attri_value.getText()); } return attributes; } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableOptionsFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableOptionsFactory.java index 325b9da697..863599ddeb 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableOptionsFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableOptionsFactory.java @@ -16,6 +16,7 @@ package com.oceanbase.tools.sqlparser.adapter.mysql; import java.math.BigDecimal; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -24,12 +25,14 @@ import org.antlr.v4.runtime.ParserRuleContext; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Lob_storage_clauseContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Parallel_optionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Table_optionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Table_option_listContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Table_option_list_space_seperatedContext; import com.oceanbase.tools.sqlparser.obmysql.OBParserBaseVisitor; import com.oceanbase.tools.sqlparser.statement.Expression; +import com.oceanbase.tools.sqlparser.statement.common.mysql.LobStorageOption; import com.oceanbase.tools.sqlparser.statement.createtable.TableOptions; import com.oceanbase.tools.sqlparser.statement.expression.BoolValue; import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; @@ -172,6 +175,8 @@ public TableOptions visitTable_option(Table_optionContext ctx) { .map(ex -> new MySQLExpressionFactory(ex).generate()) .collect(Collectors.toList()); value = new CollectionExpression(e.expr_list(), exprs); + } else if (e.compression_name() != null) { + value = new ConstExpression(e.compression_name()); } formatMap.put(e.format_key.getText().toUpperCase(), value); }); @@ -187,6 +192,60 @@ public TableOptions visitTable_option(Table_optionContext ctx) { target.setDefaultLobInRowThreshold(Integer.valueOf(ctx.INTNUM().getText())); } else if (ctx.LOB_INROW_THRESHOLD() != null) { target.setLobInRowThreshold(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.KEY_BLOCK_SIZE() != null) { + target.setKeyBlockSize(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.AUTO_INCREMENT_CACHE_SIZE() != null) { + target.setAutoIncrementCacheSize(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.PARTITION_TYPE() != null) { + target.setPartitionType(ctx.USER_SPECIFIED().getText()); + } else if (ctx.PROPERTIES() != null) { + Map externalProperties = new HashMap<>(); + ctx.external_properties_list().external_properties().forEach(e -> { + externalProperties.put(e.external_properties_key().getText(), e.STRING_VALUE().getText()); + }); + target.setExternalProperties(externalProperties); + } else if (ctx.lob_storage_clause() != null) { + target.setLobStorageOption(getLobStorageOption(ctx.lob_storage_clause())); + } else if (ctx.MICRO_INDEX_CLUSTERED() != null) { + target.setMicroIndexClustered(Boolean.valueOf(ctx.BOOL_VALUE().getText())); + } else if (ctx.AUTO_REFRESH() != null) { + if (ctx.OFF() != null) { + target.setAutoRefresh(ctx.OFF().getText()); + } else if (ctx.IMMEDIATE() != null) { + target.setAutoRefresh(ctx.IMMEDIATE().getText()); + } else if (ctx.INTERVAL() != null) { + target.setAutoRefresh(ctx.INTERVAL().getText()); + } + } else if (ctx.MIN_ROWS() != null) { + target.setMinRows(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.MAX_ROWS() != null) { + target.setMaxRows(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.PASSWORD() != null) { + target.setPassword(ctx.STRING_VALUE().getText()); + } else if (ctx.PACK_KEYS() != null) { + target.setPackKeys(ctx.INTNUM() != null ? ctx.INTNUM().getText() : ctx.DEFAULT().getText()); + } else if (ctx.CONNECTION() != null) { + target.setConnection(ctx.STRING_VALUE().getText()); + } else if (ctx.DATA() != null && ctx.DIRECTORY() != null) { + target.setDataDirectory(ctx.STRING_VALUE().getText()); + } else if (ctx.INDEX() != null && ctx.DIRECTORY() != null) { + target.setIndexDirectory(ctx.STRING_VALUE().getText()); + } else if (ctx.ENCRYPTION() != null) { + target.setEncryption(ctx.STRING_VALUE().getText()); + } else if (ctx.STATS_AUTO_RECALC() != null) { + target.setStatsAutoRecalc(ctx.INTNUM() != null ? ctx.INTNUM().getText() : ctx.DEFAULT().getText()); + } else if (ctx.STATS_PERSISTENT() != null) { + target.setStatsPersistent(ctx.INTNUM() != null ? ctx.INTNUM().getText() : ctx.DEFAULT().getText()); + } else if (ctx.STATS_SAMPLE_PAGES() != null) { + target.setStatsSamplePages(ctx.INTNUM() != null ? ctx.INTNUM().getText() : ctx.DEFAULT().getText()); + } else if (ctx.UNION() != null) { + target.setUnion(Collections.emptyList()); + if (ctx.table_list() != null) { + target.setUnion(ctx.table_list().relation_factor().stream() + .map(MySQLFromReferenceFactory::getRelationFactor).collect(Collectors.toList())); + } + } else if (ctx.INSERT_METHOD() != null) { + target.setInsertMethod(ctx.merge_insert_types().getText()); } return target; } @@ -202,4 +261,15 @@ public TableOptions visitParallel_option(Parallel_optionContext ctx) { return tableOptions; } + public static LobStorageOption getLobStorageOption(Lob_storage_clauseContext ctx) { + List lobStorageSizes = ctx.lob_storage_parameters().lob_storage_parameter() + .stream().map(i -> { + if (i.lob_chunk_size().INTNUM() != null) { + return i.lob_chunk_size().INTNUM().getText(); + } + return i.lob_chunk_size().STRING_VALUE().getText(); + }).collect(Collectors.toList()); + return new LobStorageOption(ctx, ctx.column_name().getText(), lobStorageSizes); + } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableActionFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableActionFactory.java index 3736972307..c77af1ee78 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableActionFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableActionFactory.java @@ -15,9 +15,7 @@ */ package com.oceanbase.tools.sqlparser.adapter.oracle; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import org.antlr.v4.runtime.CharStream; @@ -25,10 +23,12 @@ import org.antlr.v4.runtime.misc.Interval; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Add_external_table_partition_actionsContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Add_range_or_list_partitionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Add_range_or_list_subpartitionContext; -import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_column_group_optionContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_column_group_actionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_column_optionContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_external_table_actionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_index_optionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_partition_optionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_table_actionContext; @@ -43,6 +43,7 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Split_list_partitionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Split_range_partitionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParserBaseVisitor; +import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTableAction; import com.oceanbase.tools.sqlparser.statement.alter.table.PartitionSplitActions; import com.oceanbase.tools.sqlparser.statement.common.ColumnGroupElement; @@ -56,6 +57,7 @@ import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.TableOptions; import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; import lombok.NonNull; @@ -75,10 +77,14 @@ public OracleAlterTableActionFactory(@NonNull Alter_table_actionContext alterTab this.parserRuleContext = alterTableActionContext; } - public OracleAlterTableActionFactory(@NonNull Alter_column_group_optionContext alterColumnGroupOptionContext) { + public OracleAlterTableActionFactory(@NonNull Alter_column_group_actionContext alterColumnGroupOptionContext) { this.parserRuleContext = alterColumnGroupOptionContext; } + public OracleAlterTableActionFactory(@NonNull Alter_external_table_actionContext alterExternalTableActionContext) { + this.parserRuleContext = alterExternalTableActionContext; + } + @Override public AlterTableAction generate() { return visit(this.parserRuleContext); @@ -130,6 +136,20 @@ public AlterTableAction visitOpt_alter_compress_option(Opt_alter_compress_option return alterTableAction; } + @Override + public AlterTableAction visitAlter_external_table_action(Alter_external_table_actionContext ctx) { + AlterTableAction action = new AlterTableAction(ctx); + action.setExternalTableLocation(ctx.STRING_VALUE().getText()); + if (ctx.DROP() != null && ctx.PARTITION() != null) { + action.setDropExternalTablePartition(true); + } else if (ctx.ADD() != null && ctx.PARTITION() != null) { + Map externalTablePartition = new HashMap<>(); + visitAddExternalTablePartitionActions(externalTablePartition, ctx.add_external_table_partition_actions()); + action.setAddExternalTablePartition(externalTablePartition); + } + return action; + } + @Override public AlterTableAction visitAlter_column_option(Alter_column_optionContext ctx) { AlterTableAction alterTableAction = new AlterTableAction(ctx); @@ -248,6 +268,9 @@ public AlterTableAction visitAlter_partition_option(Alter_partition_optionContex .collect(Collectors.toList()); } alterTableAction.addSubpartitionElements(getRelationFactor(ctx.relation_factor()), subElts); + } else if (ctx.EXCHANGE() != null && ctx.PARTITION() != null) { + alterTableAction.setExchangePartition(ctx.relation_name(0).getText(), + OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor())); } else if (ctx.add_range_or_list_partition() != null) { Add_range_or_list_partitionContext pCtx = ctx.add_range_or_list_partition(); List elts; @@ -329,7 +352,7 @@ public AlterTableAction visitModify_partition_info(Modify_partition_infoContext } @Override - public AlterTableAction visitAlter_column_group_option(Alter_column_group_optionContext ctx) { + public AlterTableAction visitAlter_column_group_action(Alter_column_group_actionContext ctx) { AlterTableAction action = new AlterTableAction(ctx); List columnGroupElements = ctx.column_group_list().column_group_element() .stream().map(c -> new OracleColumnGroupElementFactory(c).generate()).collect(Collectors.toList()); @@ -379,4 +402,14 @@ private List getSpecialPartitionElement(Special_partition_list }).collect(Collectors.toList()); } + private void visitAddExternalTablePartitionActions(Map externalTablePartition, + Add_external_table_partition_actionsContext context) { + if (context == null) { + return; + } + Expression value = new ConstExpression(context.add_external_table_partition_action().expr_const()); + externalTablePartition.put(context.add_external_table_partition_action().column_name().getText(), value); + visitAddExternalTablePartitionActions(externalTablePartition, context.add_external_table_partition_actions()); + } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableFactory.java index e3c3bc5851..580ddb5ba6 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableFactory.java @@ -58,9 +58,12 @@ public AlterTable visitAlter_table_stmt(Alter_table_stmtContext ctx) { List actions = ctx.alter_table_actions().alter_table_action().stream() .map(c -> new OracleAlterTableActionFactory(c).generate()).collect(Collectors.toList()); alterTable = new AlterTable(ctx, relationFactor, actions); + } else if (ctx.alter_external_table_action() != null) { + alterTable = new AlterTable(ctx, relationFactor, Collections.singletonList( + new OracleAlterTableActionFactory(ctx.alter_external_table_action()).generate())); } else { alterTable = new AlterTable(ctx, relationFactor, Collections.singletonList( - new OracleAlterTableActionFactory(ctx.alter_column_group_option()).generate())); + new OracleAlterTableActionFactory(ctx.alter_column_group_action()).generate())); } if (ctx.EXTERNAL() != null) { alterTable.setExternal(true); diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleCreateIndexFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleCreateIndexFactory.java index 8b85428350..64638fe736 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleCreateIndexFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleCreateIndexFactory.java @@ -57,6 +57,9 @@ public CreateIndex visitCreate_index_stmt(Create_index_stmtContext ctx) { CreateIndex index = new CreateIndex(ctx, OracleFromReferenceFactory.getRelationFactor(ctx.normal_relation_factor()), OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()), columns); + if (ctx.INDEXTYPE() != null && ctx.MDSYS() != null && ctx.SPATIAL_INDEX() != null) { + index.setMdSysDotSpatialIndex(true); + } if (ctx.opt_index_options() != null) { IndexOptions options = new OracleIndexOptionsFactory(ctx.opt_index_options()).generate(); Index_using_algorithmContext context = ctx.index_using_algorithm(); diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleDataTypeFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleDataTypeFactory.java index 137c18e030..2ff47c98cc 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleDataTypeFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleDataTypeFactory.java @@ -255,6 +255,8 @@ public boolean isBinary() { public DataType visitTreat_data_type(Treat_data_typeContext ctx) { if (ctx.JSON() != null) { return new GeneralDataType(ctx, ctx.JSON().getText(), null); + } else if (ctx.obj_access_ref_cast() != null) { + return new GeneralDataType(ctx, ctx.obj_access_ref_cast().getText(), null); } return visitChildren(ctx); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleExpressionFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleExpressionFactory.java index 0166b88715..75bc21a2a4 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleExpressionFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleExpressionFactory.java @@ -51,11 +51,14 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Dot_notation_pathContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Dot_notation_path_obj_access_refContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Entry_opContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Environment_id_functionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Evalname_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.ExprContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Extract_functionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Func_access_refContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Func_paramContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Func_param_with_assignContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Hierarchical_functionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.In_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Insert_child_xmlContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Is_json_constrainContext; @@ -87,6 +90,7 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Json_value_on_optContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Json_value_on_responseContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Nstring_length_iContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Numeric_functionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Obj_access_refContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Obj_access_ref_normalContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Opt_js_value_returning_typeContext; @@ -103,9 +107,12 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Regular_entry_objContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Relation_nameContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Scalars_optContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Sdo_relate_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Signed_literalContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Simple_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Single_row_functionContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Spatial_cellid_exprContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Spatial_mbr_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Special_func_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.String_length_iContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Table_element_access_listContext; @@ -150,14 +157,14 @@ import com.oceanbase.tools.sqlparser.statement.expression.FullTextSearch; import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; import com.oceanbase.tools.sqlparser.statement.expression.FunctionParam; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.ScalarsMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.StrictMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.UniqueMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.WrapperMode; import com.oceanbase.tools.sqlparser.statement.expression.JsonKeyValue; import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption; import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption.OnMismatch; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.ScalarsMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.StrictMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.UniqueMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.WrapperMode; import com.oceanbase.tools.sqlparser.statement.expression.NullExpression; import com.oceanbase.tools.sqlparser.statement.expression.ParamWithAssign; import com.oceanbase.tools.sqlparser.statement.expression.RelationReference; @@ -666,7 +673,7 @@ public Expression visitBool_pri_in_pl_func(Bool_pri_in_pl_funcContext ctx) { @Override public Expression visitIs_json_constrain(Is_json_constrainContext ctx) { - JsonConstraint constraint = new JsonConstraint(ctx); + JsonOption constraint = new JsonOption(ctx); if (ctx.strict_opt() != null) { constraint.setStrictMode(ctx.strict_opt().LAX() != null ? StrictMode.LAX : StrictMode.STRICT); } @@ -699,42 +706,49 @@ public Expression visitJson_object_expr(Json_object_exprContext ctx) { FunctionCall fCall = new FunctionCall(ctx, "json_object", params); if (ctx.opt_json_object_content().opt_json_object_clause() != null) { Opt_json_object_clauseContext oCtx = ctx.opt_json_object_content().opt_json_object_clause(); + JsonOption jsonOpt = null; + if (oCtx.STRICT() != null || oCtx.json_obj_unique_key() != null) { + jsonOpt = getJsonOption(oCtx.STRICT(), oCtx.json_obj_unique_key()); + } if (oCtx.js_on_null() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(oCtx.js_on_null()); + } JsonOnOption onOption = new JsonOnOption(oCtx.js_on_null()); if (oCtx.js_on_null().ABSENT() != null) { onOption.setOnNull(new ConstExpression(oCtx.js_on_null().ABSENT())); } else { onOption.setOnNull(new NullExpression(oCtx.js_on_null().NULLX(0))); } - fCall.addOption(onOption); + jsonOpt.setOnOption(onOption); } if (oCtx.json_obj_returning_type() != null) { fCall.addOption(new OracleDataTypeFactory( oCtx.json_obj_returning_type().js_return_type()).generate()); } - if (oCtx.STRICT() != null || oCtx.json_obj_unique_key() != null) { - fCall.addOption(getJsonConstraint(oCtx.STRICT(), oCtx.json_obj_unique_key())); + if (jsonOpt != null) { + fCall.addOption(jsonOpt); } } else if (ctx.opt_json_object_content().STRICT() != null) { - fCall.addOption(getJsonConstraint(ctx.opt_json_object_content().STRICT(), + fCall.addOption(getJsonOption(ctx.opt_json_object_content().STRICT(), ctx.opt_json_object_content().json_obj_unique_key())); } else { - fCall.addOption(getJsonConstraint(null, ctx.opt_json_object_content().json_obj_unique_key())); + fCall.addOption(getJsonOption(null, ctx.opt_json_object_content().json_obj_unique_key())); } return fCall; } - private JsonConstraint getJsonConstraint(TerminalNode strict, Json_obj_unique_keyContext ctx) { - JsonConstraint jc; + private JsonOption getJsonOption(TerminalNode strict, Json_obj_unique_keyContext ctx) { + JsonOption jc; if (strict != null && ctx != null) { - jc = new JsonConstraint(strict, ctx); + jc = new JsonOption(strict, ctx); jc.setStrictMode(StrictMode.STRICT); jc.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); } else if (strict != null) { - jc = new JsonConstraint(strict); + jc = new JsonOption(strict); jc.setStrictMode(StrictMode.STRICT); } else { - jc = new JsonConstraint(ctx); + jc = new JsonOption(ctx); jc.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); } return jc; @@ -781,23 +795,28 @@ public Expression visitJson_query_expr(Json_query_exprContext ctx) { if (ctx.js_query_return_type() != null) { fCall.addOption(new OracleDataTypeFactory(ctx.js_query_return_type()).generate()); } - if (ctx.TRUNCATE() != null) { - fCall.addOption(new ConstExpression(ctx.TRUNCATE())); - } - if (ctx.PRETTY() != null) { - fCall.addOption(new ConstExpression(ctx.PRETTY())); - } - if (ctx.ASCII() != null) { - fCall.addOption(new ConstExpression(ctx.ASCII())); - } - if (ctx.scalars_opt() != null || ctx.wrapper_opts() != null) { - JsonConstraint constraint = new JsonConstraint( - ctx.scalars_opt() == null ? ctx.wrapper_opts() : ctx.scalars_opt()); - setScalarsMode(constraint, ctx.scalars_opt()); - setWrapperMode(constraint, ctx.wrapper_opts()); - fCall.addOption(constraint); + if (ctx.json_query_opt() != null) { + JsonOption jsonOpt = new JsonOption(ctx.json_query_opt()); + fCall.addOption(jsonOpt); + if (ctx.json_query_opt().TRUNCATE() != null) { + jsonOpt.setTruncate(true); + } + if (ctx.json_query_opt().PRETTY() != null) { + jsonOpt.setPretty(true); + } + if (ctx.json_query_opt().ASCII() != null) { + jsonOpt.setAscii(true); + } + setScalarsMode(jsonOpt, ctx.json_query_opt().scalars_opt()); + setWrapperMode(jsonOpt, ctx.json_query_opt().wrapper_opts()); + if (ctx.json_query_opt().ASIS() != null) { + jsonOpt.setAsis(true); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.json_query_opt().json_query_on_opt())); + if (ctx.json_query_opt().MULTIVALUE() != null) { + jsonOpt.setMultiValue(true); + } } - fCall.addOption(getJsonOnOption(ctx.json_query_on_opt())); return fCall; } @@ -809,24 +828,27 @@ public Expression visitJson_mergepatch_expr(Json_mergepatch_exprContext ctx) { if (ctx.js_mp_return_clause() != null) { fCall.addOption(new OracleDataTypeFactory(ctx.js_mp_return_clause().js_return_type()).generate()); } - Opt_json_mergepatchContext oCtx = ctx.opt_json_mergepatch(); + JsonOption jsonOpt = new JsonOption(ctx.json_mergepatch_opt()); + fCall.addOption(jsonOpt); + Opt_json_mergepatchContext oCtx = ctx.json_mergepatch_opt().opt_json_mergepatch(); if (oCtx.TRUNCATE() != null) { - fCall.addOption(new ConstExpression(oCtx.TRUNCATE())); + jsonOpt.setTruncate(true); } if (oCtx.PRETTY() != null) { - fCall.addOption(new ConstExpression(oCtx.PRETTY())); + jsonOpt.setPretty(true); } if (oCtx.ASCII() != null) { - fCall.addOption(new ConstExpression(oCtx.ASCII())); + jsonOpt.setAscii(true); } - if (ctx.json_mergepatch_on_error() != null) { - JsonOnOption jsonOnOption = new JsonOnOption(ctx.json_mergepatch_on_error()); - if (ctx.json_mergepatch_on_error().NULLX() != null) { - jsonOnOption.setOnError(new NullExpression(ctx.json_mergepatch_on_error().NULLX())); + if (ctx.json_mergepatch_opt().json_mergepatch_on_error() != null) { + OBParser.Json_mergepatch_on_errorContext jCtx = ctx.json_mergepatch_opt().json_mergepatch_on_error(); + JsonOnOption jsonOnOption = new JsonOnOption(jCtx); + if (jCtx.NULLX() != null) { + jsonOnOption.setOnError(new NullExpression(jCtx.NULLX())); } else { - jsonOnOption.setOnError(new ConstExpression(ctx.json_mergepatch_on_error().ERROR_P(0))); + jsonOnOption.setOnError(new ConstExpression(jCtx.ERROR_P(0))); } - fCall.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); } return fCall; } @@ -846,22 +868,28 @@ public Expression visitJson_array_expr(Json_array_exprContext ctx) { return p; }).collect(Collectors.toList()); FunctionCall fCall = new FunctionCall(ctx, "json_array", params); + JsonOption jsonOpt = null; if (jCtx.json_array_on_null() != null) { + jsonOpt = new JsonOption(jCtx.json_array_on_null()); JsonOnOption jsonOnOption = new JsonOnOption(jCtx.json_array_on_null()); if (jCtx.json_array_on_null().ABSENT() != null) { jsonOnOption.setOnNull(new ConstExpression(jCtx.json_array_on_null().ABSENT())); } else { jsonOnOption.setOnNull(new NullExpression(jCtx.json_array_on_null().NULLX(0))); } - fCall.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); } if (jCtx.js_array_return_clause() != null) { fCall.addOption(new OracleDataTypeFactory(jCtx.js_array_return_clause().js_return_type()).generate()); } if (jCtx.STRICT() != null) { - JsonConstraint jsonConstraint = new JsonConstraint(jCtx.STRICT()); - jsonConstraint.setStrictMode(StrictMode.STRICT); - fCall.addOption(jsonConstraint); + if (jsonOpt == null) { + jsonOpt = new JsonOption(jCtx.STRICT()); + } + jsonOpt.setStrictMode(StrictMode.STRICT); + } + if (jsonOpt != null) { + fCall.addOption(jsonOpt); } return fCall; } @@ -878,13 +906,17 @@ public Expression visitJson_value_expr(Json_value_exprContext ctx) { if (dataType != null) { fCall.addOption(dataType); } - if (ctx.TRUNCATE() != null) { - fCall.addOption(new ConstExpression(ctx.TRUNCATE())); - } - if (ctx.ASCII() != null) { - fCall.addOption(new ConstExpression(ctx.ASCII())); + if (ctx.json_value_opt() != null) { + JsonOption jsonOpt = new JsonOption(ctx.json_value_opt()); + fCall.addOption(jsonOpt); + if (ctx.json_value_opt().TRUNCATE() != null) { + jsonOpt.setTruncate(true); + } + if (ctx.json_value_opt().ASCII() != null) { + jsonOpt.setAscii(true); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.json_value_opt().json_value_on_opt())); } - fCall.addOption(getJsonOnOption(ctx.json_value_on_opt())); return fCall; } @@ -912,7 +944,7 @@ public Expression visitJson_table_expr(Json_table_exprContext ctx) { params.add(new ExpressionParam(new ConstExpression(ctx.literal()))); } FunctionCall fCall = new FunctionCall(ctx, ctx.JSON_TABLE().getText(), params); - fCall.addOption(getJsonOnOption(ctx.opt_json_table_on_error_on_empty())); + fCall.addOption(getJsonOption(ctx.opt_json_table_on_error_on_empty())); ctx.json_table_columns_def_opt().json_table_columns_def().json_table_column_def() .forEach(c -> fCall.addOption(visitJsonTableColumnDef(c))); return fCall; @@ -927,7 +959,7 @@ public Expression visitJson_equal_expr(Json_equal_exprContext ctx) { } FunctionCall fCall = new FunctionCall(ctx, ctx.getChild(0).getText(), params); if (ctx.json_equal_option() != null) { - fCall.addOption(getJsonOnOption(ctx.json_equal_option())); + fCall.addOption(getJsonOption(ctx.json_equal_option())); } return fCall; } @@ -1230,7 +1262,7 @@ public FunctionCall getFunctionCall(Access_func_exprContext ctx) { } setJsonExistOpt(fCall, ctx.opt_json_exist()); if (ctx.json_equal_option() != null) { - fCall.addOption(getJsonOnOption(ctx.json_equal_option())); + fCall.addOption(getJsonOption(ctx.json_equal_option())); } return fCall; } @@ -1280,88 +1312,115 @@ public Expression visitObj_access_ref_normal(Obj_access_ref_normalContext ctx) { } @Override - public Expression visitSingle_row_function(Single_row_functionContext ctx) { + public Expression visitNumeric_function(Numeric_functionContext ctx) { + return new FunctionCall(ctx, ctx.MOD().getText(), ctx.bit_expr().stream() + .map(e -> new ExpressionParam(visit(e))).collect(Collectors.toList())); + } + + @Override + public Expression visitCharacter_function(Character_functionContext ctx) { String funcName = null; List functionOpts = new ArrayList<>(); List params = new ArrayList<>(); - if (ctx.numeric_function() != null) { - funcName = ctx.numeric_function().MOD().getText(); - params.addAll(ctx.numeric_function().bit_expr().stream() - .map(e -> new ExpressionParam(visit(e))).collect(Collectors.toList())); - } else if (ctx.character_function() != null) { - Character_functionContext characterFunc = ctx.character_function(); - if (characterFunc.TRANSLATE() != null) { - funcName = characterFunc.TRANSLATE().getText(); - } else if (characterFunc.TRIM() != null) { - funcName = characterFunc.TRIM().getText(); - } else if (characterFunc.ASCII() != null) { - funcName = characterFunc.ASCII().getText(); - } - if (characterFunc.parameterized_trim() != null) { - Parameterized_trimContext trim = characterFunc.parameterized_trim(); - FunctionParam param = new ExpressionParam(visit(trim.bit_expr(0))); - if (trim.bit_expr(1) != null) { - param.addOption(visit(trim.bit_expr(1))); - } - params.add(param); - for (int i = 0; i < trim.getChildCount(); i++) { - ParseTree p = trim.getChild(i); - if (p instanceof TerminalNode) { - functionOpts.add(new ConstExpression((TerminalNode) p)); - } else { - break; - } - } - } else { - params.addAll(characterFunc.bit_expr().stream().map(e -> new ExpressionParam(visit(e))) - .collect(Collectors.toList())); - if (params.size() > 0) { - params.get(params.size() - 1).addOption(new ConstExpression(characterFunc.translate_charset())); + if (ctx.TRANSLATE() != null) { + funcName = ctx.TRANSLATE().getText(); + } else if (ctx.TRIM() != null) { + funcName = ctx.TRIM().getText(); + } else if (ctx.ASCII() != null) { + funcName = ctx.ASCII().getText(); + } + if (funcName == null) { + throw new IllegalStateException("Missing function name"); + } + if (ctx.parameterized_trim() != null) { + Parameterized_trimContext trim = ctx.parameterized_trim(); + FunctionParam param = new ExpressionParam(visit(trim.bit_expr(0))); + if (trim.bit_expr(1) != null) { + param.addOption(visit(trim.bit_expr(1))); + } + params.add(param); + for (int i = 0; i < trim.getChildCount(); i++) { + ParseTree p = trim.getChild(i); + if (p instanceof TerminalNode) { + functionOpts.add(new ConstExpression((TerminalNode) p)); + } else { + break; } } - } else if (ctx.extract_function() != null) { - funcName = ctx.extract_function().EXTRACT().getText(); - FunctionParam p = new ExpressionParam(new ConstExpression(ctx.extract_function().date_unit_for_extract())); - p.addOption(visit(ctx.extract_function().bit_expr())); - params.add(p); - } else if (ctx.conversion_function() != null) { - Conversion_functionContext fCtx = ctx.conversion_function(); - if (fCtx.CAST() != null) { - funcName = fCtx.CAST().getText(); - FunctionParam functionParam = new ExpressionParam(visit(fCtx.bit_expr())); - functionParam.addOption(new OracleDataTypeFactory(fCtx.cast_data_type()).generate()); - params.add(functionParam); - } else { - funcName = fCtx.TREAT().getText(); - FunctionParam functionParam = new ExpressionParam(visit(fCtx.bit_expr())); - functionParam.addOption(new OracleDataTypeFactory(fCtx.treat_data_type()).generate()); - params.add(functionParam); - } - } else if (ctx.hierarchical_function() != null) { - funcName = ctx.hierarchical_function().SYS_CONNECT_BY_PATH().getText(); - params.addAll(ctx.hierarchical_function().bit_expr().stream().map(e -> new ExpressionParam(visit(e))) + } else { + params.addAll(ctx.bit_expr().stream().map(e -> new ExpressionParam(visit(e))) .collect(Collectors.toList())); - } else if (ctx.environment_id_function() != null) { - funcName = ctx.environment_id_function().getText(); - } else if (ctx.xml_function() != null) { - Expression fCall = visit(ctx.xml_function()); - if (ctx.obj_access_ref_normal() != null) { - fCall.reference(visit(ctx.obj_access_ref_normal()), ReferenceOperator.DOT); - } else if (ctx.table_element_access_list() != null) { - visitTableElementAccessList(fCall, ctx.table_element_access_list()); + if (params.size() > 0) { + params.get(params.size() - 1).addOption(new ConstExpression(ctx.translate_charset())); } - return fCall; - } else if (ctx.json_function() != null) { - return visit(ctx.json_function()); - } - if (funcName == null) { - throw new IllegalStateException("Missing function name"); } FunctionCall fCall = new FunctionCall(ctx, funcName, params); functionOpts.forEach(fCall::addOption); return fCall; } + @Override + public Expression visitExtract_function(Extract_functionContext ctx) { + FunctionParam p = new ExpressionParam(new ConstExpression(ctx.date_unit_for_extract())); + p.addOption(visit(ctx.bit_expr())); + return new FunctionCall(ctx, ctx.EXTRACT().getText(), Collections.singletonList(p)); + } + + @Override + public Expression visitConversion_function(Conversion_functionContext ctx) { + if (ctx.CAST() != null) { + FunctionParam functionParam = new ExpressionParam(visit(ctx.bit_expr())); + functionParam.addOption(new OracleDataTypeFactory(ctx.cast_data_type()).generate()); + return new FunctionCall(ctx, ctx.CAST().getText(), Collections.singletonList(functionParam)); + } + FunctionParam functionParam = new ExpressionParam(visit(ctx.bit_expr())); + functionParam.addOption(new OracleDataTypeFactory(ctx.treat_data_type()).generate()); + return new FunctionCall(ctx, ctx.TREAT().getText(), Collections.singletonList(functionParam)); + } + + @Override + public Expression visitHierarchical_function(Hierarchical_functionContext ctx) { + return new FunctionCall(ctx, ctx.SYS_CONNECT_BY_PATH().getText(), ctx.bit_expr() + .stream().map(e -> new ExpressionParam(visit(e))).collect(Collectors.toList())); + } + + @Override + public Expression visitEnvironment_id_function(Environment_id_functionContext ctx) { + return new FunctionCall(ctx, ctx.getText(), Collections.emptyList()); + } + + @Override + public Expression visitSpatial_cellid_expr(Spatial_cellid_exprContext ctx) { + return new FunctionCall(ctx, ctx.SPATIAL_CELLID().getText(), + Collections.singletonList(new ExpressionParam(visit(ctx.bit_expr())))); + } + + @Override + public Expression visitSpatial_mbr_expr(Spatial_mbr_exprContext ctx) { + return new FunctionCall(ctx, ctx.SPATIAL_MBR().getText(), + Collections.singletonList(new ExpressionParam(visit(ctx.bit_expr())))); + } + + @Override + public Expression visitSdo_relate_expr(Sdo_relate_exprContext ctx) { + return new FunctionCall(ctx, ctx.SDO_RELATE().getText(), ctx.bit_expr() + .stream().map(e -> new ExpressionParam(visit(e))).collect(Collectors.toList())); + } + + @Override + public Expression visitSingle_row_function(Single_row_functionContext ctx) { + if (ctx.xml_function() == null) { + return visitChildren(ctx); + } + Expression fCall = visit(ctx.xml_function()); + if (ctx.obj_access_ref_normal() != null) { + fCall.reference(visit(ctx.obj_access_ref_normal()), ReferenceOperator.DOT); + } else if (ctx.table_element_access_list() != null) { + visitTableElementAccessList(fCall, ctx.table_element_access_list()); + } + return fCall; + } + @Override public Expression visitAggregate_function(Aggregate_functionContext ctx) { if (ctx.funcName == null) { @@ -1452,6 +1511,9 @@ public Expression visitSpecial_func_expr(Special_func_exprContext ctx) { funcName = ctx.VALUES() == null ? ctx.DEFAULT().getText() : ctx.VALUES().getText(); StatementFactory factory = new OracleColumnRefFactory(ctx.column_definition_ref()); params.add(new ExpressionParam(factory.generate())); + } else if (ctx.LAST_REFRESH_SCN() != null) { + funcName = ctx.LAST_REFRESH_SCN().getText(); + params.add(new ExpressionParam(new ConstExpression(ctx.INTNUM()))); } else { funcName = ctx.getChild(0).getText(); params.add(new ExpressionParam(visit(ctx.bit_expr(0)))); @@ -1676,14 +1738,14 @@ public boolean isBinary() { return new GeneralDataType(ctx, ctx.RAW().getText(), null); } - private void setScalarsMode(JsonConstraint c, Scalars_optContext ctx) { + private void setScalarsMode(JsonOption c, Scalars_optContext ctx) { if (ctx == null) { return; } c.setScalarsMode(ctx.ALLOW() != null ? ScalarsMode.ALLOW_SCALARS : ScalarsMode.DISALLOW_SCALARS); } - private void setWrapperMode(JsonConstraint c, Wrapper_optsContext ctx) { + private void setWrapperMode(JsonOption c, Wrapper_optsContext ctx) { if (ctx == null) { return; } @@ -1812,28 +1874,32 @@ private JsonOnOption getJsonOnOption(Js_agg_on_nullContext ctx) { return jsonOnOption; } - private JsonOnOption getJsonOnOption(Opt_json_table_on_error_on_emptyContext ctx) { + private JsonOption getJsonOption(Opt_json_table_on_error_on_emptyContext ctx) { if (ctx == null) { return null; } + JsonOption jsonOpt = new JsonOption(ctx); JsonOnOption jsonOnOption = new JsonOnOption(ctx); + jsonOpt.setOnOption(jsonOnOption); if (ctx.json_table_on_error() != null) { jsonOnOption.setOnError(visit(ctx.json_table_on_error().json_table_on_response())); } if (ctx.json_table_on_empty() != null) { jsonOnOption.setOnEmpty(visit(ctx.json_table_on_empty().json_table_on_response())); } - return jsonOnOption; + return jsonOpt; } - private JsonOnOption getJsonOnOption(Json_equal_optionContext ctx) { + private JsonOption getJsonOption(Json_equal_optionContext ctx) { + JsonOption jsonOpt = new JsonOption(ctx); JsonOnOption jsonOnOption = new JsonOnOption(ctx); + jsonOpt.setOnOption(jsonOnOption); if (ctx.BOOL_VALUE() != null) { jsonOnOption.setOnError(new BoolValue(ctx.BOOL_VALUE())); } else { jsonOnOption.setOnError(new ConstExpression(ctx.ERROR_P(0))); } - return jsonOnOption; + return jsonOpt; } private FunctionParam visitJsonTableColumnDef(Json_table_column_defContext ctx) { @@ -1860,12 +1926,28 @@ private FunctionParam visitJsonTableExistsColumnDef(Json_table_exists_column_def FunctionParam param = new ExpressionParam(new ColumnReference( ctx.column_name(), null, null, ctx.column_name().getText())); param.addOption(new OracleDataTypeFactory(ctx.opt_jt_value_type()).generate()); + JsonOption jsonOpt = null; if (ctx.TRUNCATE() != null) { - param.addOption(new ConstExpression(ctx.TRUNCATE())); + jsonOpt = new JsonOption(ctx.TRUNCATE()); + jsonOpt.setTruncate(true); } param.addOption(new ConstExpression(ctx.EXISTS())); param.addOption(visit(ctx.json_table_column_def_path())); - param.addOption(getJsonOnOption(ctx.opt_json_exists_on_error_on_empty())); + if (ctx.ASIS() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.ASIS()); + } + jsonOpt.setAsis(true); + } + if (ctx.opt_json_exists_on_error_on_empty() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.opt_json_exists_on_error_on_empty()); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.opt_json_exists_on_error_on_empty())); + } + if (jsonOpt != null) { + param.addOption(jsonOpt); + } return param; } @@ -1880,24 +1962,40 @@ private FunctionParam visitJsonTableQueryColumnDef(Json_table_query_column_defCo } else if (ctx.JSON() != null) { param.addOption(new GeneralDataType(ctx.JSON(), ctx.JSON().getText(), null)); } + JsonOption jsonOpt = null; if (ctx.TRUNCATE() != null) { - param.addOption(new ConstExpression(ctx.TRUNCATE())); + jsonOpt = new JsonOption(ctx.TRUNCATE()); + jsonOpt.setTruncate(true); } if (ctx.scalars_opt() != null || ctx.wrapper_opts() != null) { - JsonConstraint jsonConstraint; - if (ctx.scalars_opt() != null && ctx.wrapper_opts() != null) { - jsonConstraint = new JsonConstraint(ctx.scalars_opt(), ctx.wrapper_opts()); - } else if (ctx.wrapper_opts() != null) { - jsonConstraint = new JsonConstraint(ctx.wrapper_opts()); - } else { - jsonConstraint = new JsonConstraint(ctx.scalars_opt()); + if (jsonOpt == null) { + if (ctx.scalars_opt() != null && ctx.wrapper_opts() != null) { + jsonOpt = new JsonOption(ctx.scalars_opt(), ctx.wrapper_opts()); + } else if (ctx.wrapper_opts() != null) { + jsonOpt = new JsonOption(ctx.wrapper_opts()); + } else { + jsonOpt = new JsonOption(ctx.scalars_opt()); + } } - setScalarsMode(jsonConstraint, ctx.scalars_opt()); - setWrapperMode(jsonConstraint, ctx.wrapper_opts()); - param.addOption(jsonConstraint); + setScalarsMode(jsonOpt, ctx.scalars_opt()); + setWrapperMode(jsonOpt, ctx.wrapper_opts()); } param.addOption(visit(ctx.json_table_column_def_path())); - param.addOption(getJsonOnOption(ctx.json_query_on_opt())); + if (ctx.ASIS() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.ASIS()); + } + jsonOpt.setAsis(true); + } + if (ctx.json_query_on_opt() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.json_query_on_opt()); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.json_query_on_opt())); + } + if (jsonOpt != null) { + param.addOption(jsonOpt); + } return param; } @@ -1905,11 +2003,27 @@ private FunctionParam visitJsonTableValueColumnDef(Json_table_value_column_defCo FunctionParam param = new ExpressionParam(new ColumnReference( ctx.column_name(), null, null, ctx.column_name().getText())); param.addOption(new OracleDataTypeFactory(ctx.opt_jt_value_type()).generate()); + JsonOption jsonOpt = null; if (ctx.TRUNCATE() != null) { - param.addOption(new ConstExpression(ctx.TRUNCATE())); + jsonOpt = new JsonOption(ctx.TRUNCATE()); + jsonOpt.setTruncate(true); } param.addOption(visit(ctx.json_table_column_def_path())); - param.addOption(getJsonOnOption(ctx.json_value_on_opt())); + if (ctx.ASIS() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.ASIS()); + } + jsonOpt.setAsis(true); + } + if (ctx.json_value_on_opt() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.json_value_on_opt()); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.json_value_on_opt())); + } + if (jsonOpt != null) { + param.addOption(jsonOpt); + } return param; } @@ -1930,7 +2044,11 @@ private void setJsonExistOpt(@NonNull FunctionCall functionCall, Opt_json_existC .map(c -> new ExpressionParam(visit(c.bit_expr()), c.sql_var_name().getText())) .forEach(functionCall::addOption); } - functionCall.addOption(getJsonOnOption(ctx.opt_json_exists_on_error_on_empty())); + if (ctx.opt_json_exists_on_error_on_empty() != null) { + JsonOption jsonOpt = new JsonOption(ctx.opt_json_exists_on_error_on_empty()); + functionCall.addOption(jsonOpt); + jsonOpt.setOnOption(getJsonOnOption(ctx.opt_json_exists_on_error_on_empty())); + } } private void setFunctionOptions(FunctionCall functionCall, Aggregate_functionContext ctx) { @@ -1947,7 +2065,16 @@ private void setFunctionOptions(FunctionCall functionCall, Aggregate_functionCon if (ctx.WITHIN() == null && ctx.DENSE_RANK() == null && ctx.order_by() != null) { functionCall.addOption(new OracleOrderByFactory(ctx.order_by()).generate()); } - functionCall.addOption(getJsonOnOption(ctx.js_agg_on_null())); + JsonOption jsonOpt = null; + if (ctx.STRICT() != null || ctx.json_obj_unique_key() != null) { + jsonOpt = getJsonOption(ctx.STRICT(), ctx.json_obj_unique_key()); + } + if (ctx.js_agg_on_null() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.js_agg_on_null()); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.js_agg_on_null())); + } if (ctx.js_agg_returning_type_opt() != null) { Js_agg_returning_type_optContext jCtx = ctx.js_agg_returning_type_opt(); if (jCtx.js_return_type() != null) { @@ -1956,8 +2083,8 @@ private void setFunctionOptions(FunctionCall functionCall, Aggregate_functionCon functionCall.addOption(new OracleDataTypeFactory(jCtx.js_agg_returning_type()).generate()); } } - if (ctx.STRICT() != null || ctx.json_obj_unique_key() != null) { - functionCall.addOption(getJsonConstraint(ctx.STRICT(), ctx.json_obj_unique_key())); + if (jsonOpt != null) { + functionCall.addOption(jsonOpt); } } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleFromReferenceFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleFromReferenceFactory.java index 64ed4ead8c..244a97507c 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleFromReferenceFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleFromReferenceFactory.java @@ -165,7 +165,11 @@ public FromReference visitTable_factor(Table_factorContext ctx) { if (ctx.tbl_name() != null) { return visit(ctx.tbl_name()); } else if (ctx.table_subquery() != null) { - return visit(ctx.table_subquery()); + ExpressionReference reference = (ExpressionReference) visit(ctx.table_subquery()); + if (ctx.LATERAL() != null) { + reference.setLateral(true); + } + return reference; } else if (ctx.table_reference() != null) { return visit(ctx.table_reference()); } else if (ctx.simple_expr() != null) { diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleInsertFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleInsertFactory.java index 5cf2e67530..ee9eebda35 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleInsertFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleInsertFactory.java @@ -20,6 +20,8 @@ import java.util.List; import java.util.stream.Collectors; +import org.antlr.v4.runtime.tree.TerminalNode; + import com.oceanbase.tools.sqlparser.adapter.StatementFactory; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Condition_insert_clauseContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Conditional_insert_clauseContext; @@ -130,15 +132,16 @@ public Insert visitConditional_insert_clause(Conditional_insert_clauseContext ct public Insert visitSingle_table_insert(Single_table_insertContext ctx) { InsertTable insertTable; Insert_table_clauseContext iCtx = ctx.insert_table_clause(); + TerminalNode beginNode = ctx.INTO() == null ? ctx.OVERWRITE() : ctx.INTO(); if (iCtx.dml_table_name() != null) { Dml_table_nameContext dCtx = iCtx.dml_table_name(); - insertTable = new InsertTable(ctx.INTO(), ctx.values_clause(), OracleFromReferenceFactory + insertTable = new InsertTable(beginNode, ctx.values_clause(), OracleFromReferenceFactory .getRelationFactor(dCtx.relation_factor())); if (dCtx.use_partition() != null) { insertTable.setPartitionUsage(new OraclePartitionUsageFactory(dCtx.use_partition()).generate()); } } else if (iCtx.select_with_parens() != null) { - insertTable = new InsertTable(ctx.INTO(), ctx.values_clause(), + insertTable = new InsertTable(beginNode, ctx.values_clause(), new OracleSelectBodyFactory(iCtx.select_with_parens()).generate()); } else { OracleSelectBodyFactory factory = new OracleSelectBodyFactory(iCtx.subquery()); @@ -153,7 +156,7 @@ public Insert visitSingle_table_insert(Single_table_insertContext ctx) { if (oCtx.with_check_option() != null) { select.getLastSelectBody().setWithCheckOption(true); } - insertTable = new InsertTable(ctx.INTO(), ctx.values_clause(), select); + insertTable = new InsertTable(beginNode, ctx.values_clause(), select); } if (iCtx.relation_name() != null) { insertTable.setAlias(iCtx.relation_name().getText()); @@ -192,6 +195,9 @@ public Insert visitSingle_table_insert(Single_table_insertContext ctx) { insert.setLogErrors(new OracleLogErrorsFactory(rCtx.log_error_clause()).generate()); } } + if (ctx.OVERWRITE() != null) { + insert.setOverwrite(true); + } return insert; } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OraclePartitionUsageFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OraclePartitionUsageFactory.java index 196671830a..239a31843c 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OraclePartitionUsageFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OraclePartitionUsageFactory.java @@ -16,12 +16,17 @@ package com.oceanbase.tools.sqlparser.adapter.oracle; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.External_table_partitionsContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Name_listContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Use_partitionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParserBaseVisitor; +import com.oceanbase.tools.sqlparser.statement.Expression; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; import com.oceanbase.tools.sqlparser.statement.select.PartitionType; import com.oceanbase.tools.sqlparser.statement.select.PartitionUsage; @@ -54,9 +59,24 @@ public PartitionUsage visitUse_partition(Use_partitionContext ctx) { if (ctx.SUBPARTITION() != null) { type = PartitionType.SUB_PARTITION; } - List nameList = new ArrayList<>(); - visitNameList(ctx.name_list(), nameList); - return new PartitionUsage(ctx, type, nameList); + if (ctx.name_list() != null) { + List nameList = new ArrayList<>(); + visitNameList(ctx.name_list(), nameList); + return new PartitionUsage(ctx, type, nameList); + } + Map externalTablePartition = new HashMap<>(); + visitExternalTablePartitions(ctx.external_table_partitions(), externalTablePartition); + return new PartitionUsage(ctx, type, externalTablePartition); + } + + private void visitExternalTablePartitions(External_table_partitionsContext ctx, + Map externalTablePartition) { + if (ctx == null) { + return; + } + externalTablePartition.put(ctx.external_table_partition().relation_name().getText(), + new ConstExpression(ctx.external_table_partition().expr_const())); + visitExternalTablePartitions(ctx.external_table_partitions(), externalTablePartition); } private void visitNameList(Name_listContext ctx, List nameList) { diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableElementFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableElementFactory.java index 86bc36bf41..4b23b1c30f 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableElementFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableElementFactory.java @@ -387,6 +387,8 @@ private ColumnAttributes visitColumnAttribute(Column_attributeContext ctx) { attributes.setOrigDefault(visitNowOrSignedLiteral(ctx.now_or_signed_literal())); } else if (ctx.ID() != null) { attributes.setId(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.SRID() != null) { + attributes.setSrid(Integer.valueOf(ctx.INTNUM().getText())); } else if (ctx.SKIP_INDEX() != null) { List skipIndexTypes = new ArrayList<>(); if (ctx.opt_skip_index_type_list() != null) { @@ -459,6 +461,8 @@ private ColumnAttributes visitGeneratedColumnAttribute(Generated_column_attribut attributes.setId(Integer.valueOf(ctx.INTNUM().getText())); } else if (ctx.COMMENT() != null) { attributes.setComment(ctx.STRING_VALUE().getText()); + } else if (ctx.SRID() != null) { + attributes.setSrid(Integer.valueOf(ctx.INTNUM().getText())); } else { String name = null; if (ctx.constraint_and_name() != null) { diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableOptionsFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableOptionsFactory.java index 8ce7cd3b93..875d7ae3e3 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableOptionsFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableOptionsFactory.java @@ -160,12 +160,32 @@ public TableOptions visitTable_option(Table_optionContext ctx) { .map(ex -> new OracleExpressionFactory(ex).generate()) .collect(Collectors.toList()); value = new CollectionExpression(e.expr_list(), exprs); + } else if (e.compression_name() != null) { + value = new ConstExpression(e.compression_name()); } formatMap.put(e.format_key.getText().toUpperCase(), value); }); target.setFormat(formatMap); } else if (ctx.PATTERN() != null) { target.setPattern(ctx.STRING_VALUE().getText()); + } else if (ctx.PROPERTIES() != null) { + Map externalProperties = new HashMap<>(); + ctx.external_properties_list().external_properties().forEach(e -> { + externalProperties.put(e.external_properties_key().getText(), e.STRING_VALUE().getText()); + }); + target.setExternalProperties(externalProperties); + } else if (ctx.PARTITION_TYPE() != null) { + target.setPartitionType(ctx.USER_SPECIFIED().getText()); + } else if (ctx.MICRO_INDEX_CLUSTERED() != null) { + target.setMicroIndexClustered(Boolean.valueOf(ctx.BOOL_VALUE().getText())); + } else if (ctx.AUTO_REFRESH() != null) { + if (ctx.OFF() != null) { + target.setAutoRefresh(ctx.OFF().getText()); + } else if (ctx.IMMEDIATE() != null) { + target.setAutoRefresh(ctx.IMMEDIATE().getText()); + } else if (ctx.INTERVAL() != null) { + target.setAutoRefresh(ctx.INTERVAL().getText()); + } } return target; } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/alter/table/AlterTableAction.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/alter/table/AlterTableAction.java index 9050193876..f1eb1d0379 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/alter/table/AlterTableAction.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/alter/table/AlterTableAction.java @@ -17,6 +17,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; @@ -26,6 +27,7 @@ import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.common.ColumnGroupElement; import com.oceanbase.tools.sqlparser.statement.common.RelationFactor; +import com.oceanbase.tools.sqlparser.statement.common.mysql.LobStorageOption; import com.oceanbase.tools.sqlparser.statement.createtable.ColumnDefinition; import com.oceanbase.tools.sqlparser.statement.createtable.ConstraintState; import com.oceanbase.tools.sqlparser.statement.createtable.OutOfLineConstraint; @@ -100,6 +102,9 @@ public class AlterTableAction extends BaseStatement { private List dropPartitionNames; private List dropSubPartitionNames; private List addPartitionElements; + private Map addExternalTablePartition; + private String externalTableLocation; + private boolean dropExternalTablePartition; @Setter(AccessLevel.NONE) private RelationFactor addSubPartitionElementTo; @Setter(AccessLevel.NONE) @@ -147,11 +152,19 @@ public class AlterTableAction extends BaseStatement { private String renameToSubPartitionName; private List addColumnGroupElements; private List dropColumnGroupElements; + private String exchangePartitionName; + private RelationFactor exchangePartitionTargetTable; + private LobStorageOption lobStorageOption; public AlterTableAction(@NonNull ParserRuleContext context) { super(context); } + public void setExchangePartition(@NonNull String partitionName, @NonNull RelationFactor targetTable) { + this.exchangePartitionName = partitionName; + this.exchangePartitionTargetTable = targetTable; + } + public void setDropColumn(@NonNull ColumnReference dropColumn, String dropColumnOption) { this.dropColumns = Collections.singletonList(dropColumn); @@ -249,6 +262,9 @@ public String toString() { .map(ColumnDefinition::toString) .collect(Collectors.joining(","))) .append(")"); + if (this.lobStorageOption != null) { + builder.append(" ").append(this.lobStorageOption); + } } } if (CollectionUtils.isNotEmpty(this.dropColumns)) { @@ -410,6 +426,10 @@ public String toString() { if (Boolean.TRUE.equals(this.removePartitioning)) { builder.append(" REMOVE PARTITIONING"); } + if (this.exchangePartitionTargetTable != null && this.exchangePartitionName != null) { + builder.append(" EXCHANGE PARTITION ").append(this.exchangePartitionName) + .append(" WITH TABLE ").append(this.exchangePartitionTargetTable).append(" WITHOUT VALIDATION"); + } if (addColumnGroupElements != null) { builder.append(" ADD COLUMN GROUP(") .append(addColumnGroupElements.stream().map(ColumnGroupElement::toString) @@ -422,6 +442,16 @@ public String toString() { .collect(Collectors.joining(","))) .append(")"); } + if (this.dropExternalTablePartition) { + builder.append(" DROP PARTITION"); + } + if (this.addExternalTablePartition != null) { + builder.append(" ADD PARTITION(").append(this.addExternalTablePartition.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", "))).append(")"); + } + if (this.externalTableLocation != null) { + builder.append(" LOCATION ").append(this.externalTableLocation); + } return builder.length() == 0 ? "" : builder.substring(1); } @@ -453,7 +483,7 @@ public boolean isSetDefault() { @Override public String toString() { - return this.isSetDefault() ? "SET DEFAULT" + this.defaultValue : "DROP DEFAULT"; + return this.isSetDefault() ? "SET DEFAULT (" + this.defaultValue + ")" : "DROP DEFAULT"; } } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/ArrayType.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/ArrayType.java index d6567dca70..9f907d807c 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/ArrayType.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/ArrayType.java @@ -37,7 +37,7 @@ @Setter @EqualsAndHashCode(callSuper = false) public class ArrayType extends BaseStatement implements DataType { - private final String typeName = "ARRAY"; + private final DataType elementType; public ArrayType(@NonNull ParserRuleContext context, @NonNull DataType elementType) { @@ -45,17 +45,23 @@ public ArrayType(@NonNull ParserRuleContext context, @NonNull DataType elementTy this.elementType = elementType; } - public ArrayType(DataType elementType) { + public ArrayType(@NonNull DataType elementType) { this.elementType = elementType; } @Override public String getName() { - return this.typeName; + return "ARRAY"; } @Override public List getArguments() { return Collections.emptyList(); } + + @Override + public String toString() { + return getName() + "(" + this.elementType + ")"; + } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/LobStorageOption.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/LobStorageOption.java new file mode 100644 index 0000000000..e6b86c2a3f --- /dev/null +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/LobStorageOption.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.tools.sqlparser.statement.common.mysql; + +import java.util.List; +import java.util.stream.Collectors; + +import org.antlr.v4.runtime.ParserRuleContext; + +import com.oceanbase.tools.sqlparser.statement.BaseStatement; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NonNull; + +/** + * {@link LobStorageOption} + * + * @author yh263208 + * @date 2024-10-25 15:29 + * @since ODC_release_4.3.2 + */ +@Getter +@EqualsAndHashCode(callSuper = false) +public class LobStorageOption extends BaseStatement { + + private final String columnName; + private final List lobChunkSizes; + + public LobStorageOption(@NonNull ParserRuleContext context, + @NonNull String columnName, @NonNull List lobChunkSizes) { + super(context); + this.columnName = columnName; + this.lobChunkSizes = lobChunkSizes; + } + + public LobStorageOption(@NonNull String columnName, @NonNull List lobChunkSizes) { + this.columnName = columnName; + this.lobChunkSizes = lobChunkSizes; + } + + @Override + public String toString() { + String lobChunkSize = this.lobChunkSizes.stream().map(s -> "CHUNK " + s).collect(Collectors.joining(" ")); + return "JSON(" + this.columnName + ")" + " STORE AS (" + lobChunkSize + ")"; + } + +} diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/VectorType.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/VectorType.java index fe16933523..29a895291a 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/VectorType.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/VectorType.java @@ -37,6 +37,7 @@ @Setter @EqualsAndHashCode(callSuper = false) public class VectorType extends BaseStatement implements DataType { + private final String typeName; private final Integer dimension; @@ -64,8 +65,7 @@ public List getArguments() { @Override public String toString() { - StringBuilder builder = new StringBuilder(getName()); - builder.append("(").append(dimension).append(")"); - return builder.toString(); + return getName() + "(" + this.dimension + ")"; } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createindex/CreateIndex.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createindex/CreateIndex.java index 0dd3035a70..156972b160 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createindex/CreateIndex.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createindex/CreateIndex.java @@ -53,6 +53,7 @@ public class CreateIndex extends BaseStatement { private RelationFactor relation; private IndexOptions indexOptions; private Partition partition; + private boolean mdSysDotSpatialIndex; private final List columns; private List columnGroupElements; @@ -96,6 +97,9 @@ public String toString() { .append(" (\n\t").append(this.columns.stream() .map(SortColumn::toString).collect(Collectors.joining(",\n\t"))) .append("\n)"); + if (this.mdSysDotSpatialIndex) { + builder.append(" INDEXTYPE IS MDSYS.SPATIAL_INDEX"); + } if (this.indexOptions != null) { builder.append(" ").append(this.indexOptions); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnAttributes.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnAttributes.java index 7067779422..2ad06647f2 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnAttributes.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnAttributes.java @@ -56,6 +56,9 @@ public class ColumnAttributes extends BaseOptions { private Expression onUpdate; private String collation; private Integer srid; + private String lobChunkSize; + private String columnFormat; + private String storage; private List constraints; private List skipIndexTypes; @@ -130,6 +133,15 @@ public String toString() { .append(String.join(",", skipIndexTypes)) .append(")"); } + if (this.lobChunkSize != null) { + builder.append(" CHUNK ").append(this.lobChunkSize); + } + if (this.columnFormat != null) { + builder.append(" COLUMN_FORMAT ").append(this.columnFormat); + } + if (this.storage != null) { + builder.append(" STORAGE ").append(this.storage); + } return builder.length() == 0 ? "" : builder.substring(1); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnDefinition.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnDefinition.java index e70bf9ac86..d9fcfcfcbb 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnDefinition.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnDefinition.java @@ -43,7 +43,12 @@ public class ColumnDefinition extends BaseStatement implements TableElement { private ColumnAttributes columnAttributes; private Boolean visible; private Location location; + /** + * if this field is true, means `BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE` + */ + private boolean serial; private GenerateOption generateOption; + private ForeignReference foreignReference; private final DataType dataType; private final ColumnReference columnReference; @@ -64,6 +69,8 @@ public String toString() { StringBuilder builder = new StringBuilder(this.columnReference.toString()); if (this.dataType != null) { builder.append(" ").append(this.dataType.toString()); + } else if (this.serial) { + builder.append(" SERIAL"); } if (this.visible != null) { if (this.visible) { @@ -78,6 +85,9 @@ public String toString() { if (this.columnAttributes != null) { builder.append(" ").append(this.columnAttributes); } + if (this.foreignReference != null) { + builder.append(" ").append(this.foreignReference); + } if (this.location != null) { builder.append(" ").append(this.location); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/CreateTable.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/CreateTable.java index 601229d1cc..a9a1f029e0 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/CreateTable.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/CreateTable.java @@ -46,6 +46,8 @@ public class CreateTable extends BaseStatement { private RelationFactor relation; + private boolean ignore; + private boolean replace; private String userVariable; private boolean global; private boolean temporary; @@ -178,6 +180,12 @@ public String toString() { .map(ColumnGroupElement::toString).collect(Collectors.joining(","))) .append(")"); } + if (this.ignore) { + builder.append(" IGNORE"); + } + if (this.replace) { + builder.append(" REPLACE"); + } if (this.as != null) { builder.append(" AS ").append(this.as); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/IndexOptions.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/IndexOptions.java index a480face47..a81fe01893 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/IndexOptions.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/IndexOptions.java @@ -16,6 +16,7 @@ package com.oceanbase.tools.sqlparser.statement.createtable; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; @@ -66,6 +67,8 @@ public class IndexOptions extends BaseOptions { private Boolean reverse; private List storing; private List ctxcat; + private Integer keyBlockSize; + private Map vectorIndexParams; public IndexOptions(@NonNull ParserRuleContext context) { super(context); @@ -149,6 +152,13 @@ public String toString() { if (this.tableSpace != null) { builder.append(" TABLESPACE ").append(this.tableSpace); } + if (this.keyBlockSize != null) { + builder.append(" KEY_BLOCK_SIZE ").append(this.keyBlockSize); + } + if (this.vectorIndexParams != null) { + builder.append(" WITH (").append(this.vectorIndexParams.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", "))).append(")"); + } return builder.length() == 0 ? "" : builder.substring(1); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/OutOfLineIndex.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/OutOfLineIndex.java index d794ee4812..24f6c671a6 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/OutOfLineIndex.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/OutOfLineIndex.java @@ -69,6 +69,8 @@ public String toString() { builder = new StringBuilder("FULLTEXT KEY"); } else if (this.spatial) { builder = new StringBuilder("SPATIAL KEY"); + } else if (this.vector) { + builder = new StringBuilder("VECTOR KEY"); } else { builder = new StringBuilder("INDEX"); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/TableOptions.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/TableOptions.java index 5ccebaa8a0..3c6a229e6a 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/TableOptions.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/TableOptions.java @@ -26,6 +26,8 @@ import com.oceanbase.tools.sqlparser.statement.BaseStatement; import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.common.BaseOptions; +import com.oceanbase.tools.sqlparser.statement.common.RelationFactor; +import com.oceanbase.tools.sqlparser.statement.common.mysql.LobStorageOption; import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; import lombok.EqualsAndHashCode; @@ -96,6 +98,26 @@ public class TableOptions extends BaseOptions { private String kvAttributes; private Integer defaultLobInRowThreshold; private Integer lobInRowThreshold; + private Integer keyBlockSize; + private Integer autoIncrementCacheSize; + private String partitionType; + private Map externalProperties; + private LobStorageOption lobStorageOption; + private Boolean microIndexClustered; + private String autoRefresh; + private Integer maxRows; + private Integer minRows; + private String password; + private String packKeys; + private String connection; + private String dataDirectory; + private String indexDirectory; + private String encryption; + private String statsAutoRecalc; + private String statsPersistent; + private String statsSamplePages; + private List union; + private String insertMethod; public TableOptions(@NonNull ParserRuleContext context) { super(context); @@ -255,6 +277,71 @@ public String toString() { if (this.lobInRowThreshold != null) { builder.append(" LOB_INROW_THRESHOLD=").append(this.lobInRowThreshold); } + if (this.keyBlockSize != null) { + builder.append(" KEY_BLOCK_SIZE=").append(this.keyBlockSize); + } + if (this.autoIncrementCacheSize != null) { + builder.append(" AUTO_INCREMENT_CACHE_SIZE=").append(this.autoIncrementCacheSize); + } + if (this.partitionType != null) { + builder.append(" PARTITION_TYPE=").append(this.partitionType); + } + if (this.externalProperties != null) { + builder.append(" PROPERTIES=(") + .append(this.externalProperties.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(","))) + .append(")"); + } + if (this.lobStorageOption != null) { + builder.append(" ").append(this.lobStorageOption); + } + if (this.microIndexClustered != null) { + builder.append(" MICRO_INDEX_CLUSTERED=").append( + Boolean.TRUE.equals(this.microIndexClustered) ? "TRUE" : "FALSE"); + } + if (this.autoRefresh != null) { + builder.append(" AUTO_REFRESH=").append(this.autoRefresh); + } + if (this.maxRows != null) { + builder.append(" MAX_ROWS=").append(this.maxRows); + } + if (this.minRows != null) { + builder.append(" MIN_ROWS=").append(this.minRows); + } + if (this.password != null) { + builder.append(" PASSWORD=").append(this.password); + } + if (this.packKeys != null) { + builder.append(" PACK_KEYS=").append(this.packKeys); + } + if (this.connection != null) { + builder.append(" CONNECTION=").append(this.connection); + } + if (this.dataDirectory != null) { + builder.append(" DATA DIRECTORY=").append(this.dataDirectory); + } + if (this.indexDirectory != null) { + builder.append(" INDEX DIRECTORY=").append(this.indexDirectory); + } + if (this.encryption != null) { + builder.append(" ENCRYPTION=").append(this.encryption); + } + if (this.statsAutoRecalc != null) { + builder.append(" STATS_AUTO_RECALC=").append(this.statsAutoRecalc); + } + if (this.statsPersistent != null) { + builder.append(" STATS_PERSISTENT=").append(this.statsPersistent); + } + if (this.statsSamplePages != null) { + builder.append(" STATS_SAMPLE_PAGES=").append(this.statsSamplePages); + } + if (this.union != null) { + builder.append(" UNION=(").append(this.union.stream() + .map(RelationFactor::toString).collect(Collectors.joining(", "))).append(")"); + } + if (this.insertMethod != null) { + builder.append(" INSERT_METHOD=").append(this.insertMethod); + } return builder.length() == 0 ? "" : builder.substring(1); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/ArrayExpression.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/ArrayExpression.java index 1fd49fce70..bbc7804d4c 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/ArrayExpression.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/ArrayExpression.java @@ -15,7 +15,6 @@ */ package com.oceanbase.tools.sqlparser.statement.expression; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -35,7 +34,8 @@ @Getter @EqualsAndHashCode(callSuper = true) public class ArrayExpression extends BaseExpression { - private List expressions = new ArrayList<>(); + + private final List expressions; public ArrayExpression(@NonNull ParserRuleContext context, @NonNull List expressions) { super(context); @@ -50,4 +50,5 @@ public ArrayExpression(@NonNull List expressions) { protected String doToString() { return "[" + this.expressions.stream().map(Object::toString).collect(Collectors.joining(",")) + "]"; } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/FullTextSearch.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/FullTextSearch.java index 668c7b5212..aa2b56ec59 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/FullTextSearch.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/FullTextSearch.java @@ -38,6 +38,8 @@ public class FullTextSearch extends FunctionCall { private final String against; @Setter private TextSearchMode searchMode; + @Setter + private boolean withQueryExpansion; public FullTextSearch(@NonNull ParserRuleContext context, @NonNull List params, @NonNull String against) { @@ -59,6 +61,9 @@ public String doToString() { if (this.searchMode != null) { builder.append(" ").append(this.searchMode.getValue()); } + if (this.withQueryExpansion) { + builder.append(" WITH QUERY EXPANSION"); + } return builder.append(")").toString(); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonConstraint.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonOption.java similarity index 72% rename from libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonConstraint.java rename to libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonOption.java index 4e045fe059..48997ec171 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonConstraint.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonOption.java @@ -26,7 +26,7 @@ import lombok.Setter; /** - * {@link JsonConstraint} + * {@link JsonOption} * * @author yh263208 * @date 2023-09-26 15:09 @@ -36,44 +36,68 @@ @Setter @NoArgsConstructor @EqualsAndHashCode(callSuper = true) -public class JsonConstraint extends BaseExpression { +public class JsonOption extends BaseExpression { + private boolean truncate; + private boolean pretty; + private boolean ascii; + private boolean asis; + private boolean multiValue; + private JsonOnOption onOption; private StrictMode strictMode; private ScalarsMode scalarsMode; private UniqueMode uniqueMode; private WrapperMode wrapperMode; - public JsonConstraint(@NonNull ParserRuleContext context) { + public JsonOption(@NonNull ParserRuleContext context) { super(context); } - public JsonConstraint(@NonNull TerminalNode terminalNode) { + public JsonOption(@NonNull TerminalNode terminalNode) { super(terminalNode); } - public JsonConstraint(@NonNull TerminalNode beginNode, @NonNull ParserRuleContext endRule) { + public JsonOption(@NonNull TerminalNode beginNode, @NonNull ParserRuleContext endRule) { super(beginNode, endRule); } - public JsonConstraint(@NonNull ParserRuleContext beginRule, @NonNull ParserRuleContext endRule) { + public JsonOption(@NonNull ParserRuleContext beginRule, @NonNull ParserRuleContext endRule) { super(beginRule, endRule); } @Override protected String doToString() { - StringBuilder builder = new StringBuilder("JSON"); + StringBuilder builder = new StringBuilder(); if (this.strictMode != null) { builder.append(" ").append(this.strictMode.name()); } + if (this.truncate) { + builder.append(" TRUNCATE"); + } if (this.scalarsMode != null) { builder.append(" ").append(this.scalarsMode.name().replace("_", " ")); } + if (this.pretty) { + builder.append(" PRETTY"); + } + if (this.ascii) { + builder.append(" ASCII"); + } if (this.uniqueMode != null) { builder.append(" ").append(this.uniqueMode.name().replace("_", " ")); } if (this.wrapperMode != null) { builder.append(" ").append(this.wrapperMode.name().replace("_", " ")); } + if (this.asis) { + builder.append(" ASIS"); + } + if (this.onOption != null) { + builder.append(" ").append(this.onOption); + } + if (this.multiValue) { + builder.append(" MULTIVALUE"); + } return builder.toString(); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/insert/Insert.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/insert/Insert.java index 005fd3c735..dc1c87ab20 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/insert/Insert.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/insert/Insert.java @@ -54,6 +54,9 @@ public class Insert extends BaseStatement { private boolean first; private boolean replace; private boolean ignore; + private boolean highPriority; + private boolean lowPriority; + private boolean overwrite; private List onDuplicateKeyUpdateColumns; private final List tableInsert; private final ConditionalInsert conditionalInsert; @@ -78,6 +81,9 @@ public Insert(@NonNull ParserRuleContext context, @NonNull Insert target) { this.ignore = target.ignore; this.onDuplicateKeyUpdateColumns = target.onDuplicateKeyUpdateColumns; this.tableInsert = target.tableInsert; + this.overwrite = target.overwrite; + this.highPriority = target.highPriority; + this.lowPriority = target.lowPriority; this.conditionalInsert = target.conditionalInsert; } @@ -95,6 +101,11 @@ public String toString() { } else { builder.append("INSERT"); } + if (this.highPriority) { + builder.append(" HIGH_PRIORITY"); + } else if (this.lowPriority) { + builder.append(" LOW_PRIORITY"); + } if (this.all) { builder.append(" ALL"); } else if (this.first) { @@ -102,6 +113,9 @@ public String toString() { } else if (this.ignore) { builder.append(" IGNORE"); } + if (this.overwrite) { + builder.append(" OVERWRITE"); + } if (CollectionUtils.isNotEmpty(this.tableInsert)) { builder.append(" ").append(this.tableInsert.stream() .map(InsertTable::toString).collect(Collectors.joining("\n"))); diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/ExpressionReference.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/ExpressionReference.java index e586f48d26..c15143fefc 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/ExpressionReference.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/ExpressionReference.java @@ -51,6 +51,8 @@ public class ExpressionReference extends BaseStatement implements FromReference @Setter private FlashbackUsage flashbackUsage; @Setter + private boolean lateral; + @Setter private List aliasColumns; public ExpressionReference(@NonNull ParserRuleContext context, @@ -67,7 +69,11 @@ public ExpressionReference(@NonNull Expression target, String alias) { @Override public String toString() { - StringBuilder builder = new StringBuilder(this.target.toString()); + StringBuilder builder = new StringBuilder(); + if (this.lateral) { + builder.append("LATERAL "); + } + builder.append(this.target.toString()); if (this.flashbackUsage != null) { builder.append(" ").append(this.flashbackUsage.toString()); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/PartitionUsage.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/PartitionUsage.java index c9d0c02bc8..fbd2d6711c 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/PartitionUsage.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/PartitionUsage.java @@ -16,10 +16,13 @@ package com.oceanbase.tools.sqlparser.statement.select; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; import com.oceanbase.tools.sqlparser.statement.BaseStatement; +import com.oceanbase.tools.sqlparser.statement.Expression; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -39,16 +42,33 @@ public class PartitionUsage extends BaseStatement { private final PartitionType type; private final List nameList; + private final Map externalTablePartition; public PartitionUsage(@NonNull ParserRuleContext context, PartitionType type, @NonNull List nameList) { super(context); this.type = type; this.nameList = nameList; + this.externalTablePartition = null; } public PartitionUsage(PartitionType type, @NonNull List nameList) { this.type = type; this.nameList = nameList; + this.externalTablePartition = null; + } + + public PartitionUsage(@NonNull ParserRuleContext context, PartitionType type, + @NonNull Map externalTablePartition) { + super(context); + this.type = type; + this.nameList = null; + this.externalTablePartition = externalTablePartition; + } + + public PartitionUsage(PartitionType type, @NonNull Map externalTablePartition) { + this.type = type; + this.nameList = null; + this.externalTablePartition = externalTablePartition; } @Override @@ -59,7 +79,11 @@ public String toString() { } else { buffer.append("SUBPARTITION "); } - return buffer.append("(").append(String.join(",", nameList)).append(")").toString(); + if (this.nameList != null) { + return buffer.append("(").append(String.join(",", nameList)).append(")").toString(); + } + return buffer.append("(").append(this.externalTablePartition.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", "))).append(")").toString(); } } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/SelectBody.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/SelectBody.java index 54886df64c..5423e22d38 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/SelectBody.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/SelectBody.java @@ -54,14 +54,16 @@ public class SelectBody extends BaseExpression { private List with = new ArrayList<>(); private boolean recursive; private List groupBy = new ArrayList<>(); - boolean withRollUp; - boolean withCheckOption; + private boolean withRollUp; + private boolean withCheckOption; + private boolean approximate; private List windows = new ArrayList<>(); private Fetch fetch; private Limit limit; private ForUpdate forUpdate; private OrderBy orderBy; private boolean lockInShareMode; + private RelatedSelectBody relatedSelect; private final List> values; private final List froms; @@ -92,6 +94,7 @@ public SelectBody(@NonNull ParserRuleContext context, @NonNull SelectBody other) this.with = other.with; this.recursive = other.recursive; this.groupBy = other.groupBy; + this.lockInShareMode = other.lockInShareMode; this.withRollUp = other.withRollUp; this.withCheckOption = other.withCheckOption; this.windows = other.windows; @@ -102,6 +105,7 @@ public SelectBody(@NonNull ParserRuleContext context, @NonNull SelectBody other) this.froms = other.froms; this.selectItems = other.selectItems; this.forUpdate = other.forUpdate; + this.approximate = other.approximate; this.values = other.values; } @@ -182,6 +186,9 @@ public String doToString() { if (this.orderBy != null) { builder.append(" ").append(this.orderBy.toString()); } + if (this.approximate) { + builder.append(" APPROXIMATE"); + } if (this.fetch != null) { builder.append(" ").append(this.fetch.toString()); } diff --git a/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 b/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 index 16a1c8e18b..472619c695 100644 --- a/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 +++ b/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 @@ -323,11 +323,6 @@ simple_expr | LeftBracket expr_list RightBracket ; -search_expr - : expr_const - | func_expr - ; - expr : (NOT|USER_VARIABLE SET_VAR) expr | LeftParen expr RightParen @@ -1025,11 +1020,6 @@ reference_action | SET DEFAULT ; -opt_match_option - : MATCH match_action - | empty - ; - match_action : SIMPLE | FULL @@ -1037,8 +1027,8 @@ match_action ; column_definition - : column_definition_ref data_type opt_column_attribute_list? (REFERENCES relation_factor LeftParen column_name_list RightParen opt_match_option opt_reference_option_list)? (FIRST | (BEFORE column_name) | (AFTER column_name))? - | column_definition_ref data_type (GENERATED opt_generated_option_list)? AS LeftParen expr RightParen (VIRTUAL | STORED)? opt_generated_column_attribute_list? (REFERENCES relation_factor LeftParen column_name_list RightParen opt_match_option opt_reference_option_list)? (FIRST | (BEFORE column_name) | (AFTER column_name))? + : column_definition_ref data_type opt_column_attribute_list? references_clause? (FIRST | (BEFORE column_name) | (AFTER column_name))? + | column_definition_ref data_type (GENERATED opt_generated_option_list)? AS LeftParen expr RightParen (VIRTUAL | STORED)? opt_generated_column_attribute_list? references_clause? (FIRST | (BEFORE column_name) | (AFTER column_name))? | column_definition_ref SERIAL opt_column_attribute_list? (FIRST | (BEFORE column_name) | (AFTER column_name))? ; @@ -1283,8 +1273,8 @@ column_attribute | COLLATE collation_name | SKIP_INDEX LeftParen (skip_index_type | (opt_skip_index_type_list Comma skip_index_type))? RightParen | lob_chunk_size - | COLUMN_FORMAT (DEFAULT|FIXED|DYNAMIC) - | STORAGE (DEFAULT|DISK|MEMORY) + | COLUMN_FORMAT col_attri_value=(DEFAULT|FIXED|DYNAMIC) + | STORAGE col_attri_value=(DEFAULT|DISK|MEMORY) ; now_or_signed_literal @@ -1359,7 +1349,6 @@ table_option | AUTO_INCREMENT_CACHE_SIZE COMP_EQ? INTNUM | PARTITION_TYPE COMP_EQ? USER_SPECIFIED | PROPERTIES COMP_EQ? LeftParen external_properties_list RightParen - | lob_storage_clause | MICRO_INDEX_CLUSTERED COMP_EQ? BOOL_VALUE | AUTO_REFRESH COMP_EQ? (OFF|IMMEDIATE|INTERVAL) @@ -1672,7 +1661,11 @@ external_properties_list ; external_properties - : ((((ACCESSID|ACCESSKEY)|(ACCESSTYPE|TYPE))|((ENDPOINT|STSTOKEN)|(PROJECT_NAME|SCHEMA_NAME)))|((COMPRESSION_CODE|QUOTA_NAME)|TABLE_NAME)) COMP_EQ STRING_VALUE + : external_properties_key COMP_EQ STRING_VALUE + ; + +external_properties_key + : ((((ACCESSID|ACCESSKEY)|(ACCESSTYPE|TYPE))|((ENDPOINT|STSTOKEN)|(PROJECT_NAME|SCHEMA_NAME)))|((COMPRESSION_CODE|QUOTA_NAME)|TABLE_NAME)) ; external_file_format_list @@ -2027,7 +2020,6 @@ select_stmt : with_clause? (select_no_parens into_clause? |select_with_parens) ; - select_with_parens : LeftParen with_clause? (select_no_parens |select_with_parens) RightParen ; @@ -3488,14 +3480,17 @@ rename_table_action alter_table_stmt : ALTER EXTERNAL? TABLE relation_factor alter_table_actions? - | ALTER TABLE relation_factor alter_column_group_option - | ALTER EXTERNAL TABLE relation_factor ADD PARTITION LeftParen add_external_table_partition_actions RightParen LOCATION STRING_VALUE - | ALTER EXTERNAL TABLE relation_factor DROP PARTITION LOCATION STRING_VALUE + | ALTER TABLE relation_factor alter_column_group_action + | ALTER EXTERNAL TABLE relation_factor alter_external_table_action + ; + +alter_external_table_action + : ADD PARTITION LeftParen add_external_table_partition_actions? RightParen LOCATION STRING_VALUE + | DROP PARTITION LOCATION STRING_VALUE ; add_external_table_partition_actions : add_external_table_partition_action - | empty | add_external_table_partition_actions Comma add_external_table_partition_action ; @@ -3585,13 +3580,13 @@ visibility_option | INVISIBLE ; -alter_column_group_option +alter_column_group_action : (ADD|DROP) COLUMN GROUP LeftParen column_group_list RightParen ; alter_column_option : ADD COLUMN? column_definition - | ADD COLUMN? LeftParen column_definition_list RightParen + | ADD COLUMN? LeftParen column_definition_list RightParen lob_storage_clause? | DROP column_definition_ref (CASCADE | RESTRICT)? | DROP COLUMN column_definition_ref (CASCADE | RESTRICT)? | ALTER COLUMN? column_definition_ref alter_column_behavior @@ -4468,7 +4463,39 @@ vec_index_param_value ; json_query_expr - : JSON_QUERY LeftParen simple_expr Comma complex_string_literal (RETURNING cast_data_type)? TRUNCATE? ((ALLOW SCALARS) | (DISALLOW SCALARS))? PRETTY? ASCII? ((WITHOUT WRAPPER) | (WITHOUT ARRAY WRAPPER) | (WITH WRAPPER) | (WITH ARRAY WRAPPER) | (WITH UNCONDITIONAL WRAPPER) | (WITH CONDITIONAL WRAPPER) | (WITH UNCONDITIONAL ARRAY WRAPPER) | (WITH CONDITIONAL ARRAY WRAPPER))? ASIS? (on_empty_query | on_error_query | on_mismatch_query | (on_error_query on_empty_query) | (on_empty_query on_error_query) | (on_error_query on_mismatch_query) | (on_empty_query on_mismatch_query) | (on_error_query on_empty_query on_mismatch_query) | (on_empty_query on_error_query on_mismatch_query))? MULTIVALUE? RightParen + : JSON_QUERY LeftParen simple_expr Comma complex_string_literal (RETURNING cast_data_type)? json_query_opt RightParen + ; + +json_query_opt + : TRUNCATE? scalars_opt? PRETTY? ASCII? wrapper_opts? ASIS? json_query_on_opt? MULTIVALUE? + ; + +scalars_opt + : ALLOW SCALARS + | DISALLOW SCALARS + ; + +wrapper_opts + : WITHOUT WRAPPER + | WITHOUT ARRAY WRAPPER + | WITH WRAPPER + | WITH ARRAY WRAPPER + | WITH UNCONDITIONAL WRAPPER + | WITH CONDITIONAL WRAPPER + | WITH UNCONDITIONAL ARRAY WRAPPER + | WITH CONDITIONAL ARRAY WRAPPER + ; + +json_query_on_opt + : on_empty_query + | on_error_query + | on_mismatch_query + | on_error_query on_empty_query + | on_empty_query on_error_query + | on_error_query on_mismatch_query + | on_empty_query on_mismatch_query + | on_error_query on_empty_query on_mismatch_query + | on_empty_query on_error_query on_mismatch_query ; opt_response_query @@ -4489,7 +4516,17 @@ on_empty_query ; json_value_expr - : JSON_VALUE LeftParen simple_expr Comma complex_string_literal (RETURNING cast_data_type)? TRUNCATE? ASCII? (on_empty | on_error | (on_empty on_error))? RightParen + : JSON_VALUE LeftParen simple_expr Comma complex_string_literal (RETURNING cast_data_type)? json_value_opt RightParen + ; + +json_value_opt + : TRUNCATE? ASCII? json_value_on_opt? + ; + +json_value_on_opt + : on_empty + | on_error + | on_empty on_error ; opt_on_empty_or_error diff --git a/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBParser.g4 b/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBParser.g4 index 35e0e9df9a..fc6045b429 100644 --- a/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBParser.g4 +++ b/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBParser.g4 @@ -1590,9 +1590,7 @@ table_option | PATTERN COMP_EQ? STRING_VALUE | PARTITION_TYPE COMP_EQ? USER_SPECIFIED | MICRO_INDEX_CLUSTERED COMP_EQ? BOOL_VALUE - | AUTO_REFRESH COMP_EQ? OFF - | AUTO_REFRESH COMP_EQ? IMMEDIATE - | AUTO_REFRESH COMP_EQ? INTERVAL + | AUTO_REFRESH COMP_EQ? (OFF|IMMEDIATE|INTERVAL) ; parallel_option @@ -1923,7 +1921,11 @@ external_properties_list ; external_properties - : ((((ACCESSID|ACCESSKEY)|(ACCESSTYPE|TYPE))|((ENDPOINT|STSTOKEN)|(PROJECT_NAME|SCHEMA_NAME)))|((COMPRESSION_CODE|QUOTA_NAME)|TABLE_NAME)) COMP_EQ STRING_VALUE + : external_properties_key COMP_EQ STRING_VALUE + ; + +external_properties_key + : ((((ACCESSID|ACCESSKEY)|(ACCESSTYPE|TYPE))|((ENDPOINT|STSTOKEN)|(PROJECT_NAME|SCHEMA_NAME)))|((COMPRESSION_CODE|QUOTA_NAME)|TABLE_NAME)) ; external_file_format_list @@ -2892,12 +2894,11 @@ table_subquery use_partition : (PARTITION|SUBPARTITION) LeftParen name_list RightParen - | PARTITION LeftParen external_table_partitions RightParen + | PARTITION LeftParen external_table_partitions? RightParen ; external_table_partitions : external_table_partition - | empty | external_table_partitions Comma external_table_partition ; @@ -3704,14 +3705,17 @@ alter_index_option_oracle alter_table_stmt : ALTER EXTERNAL? TABLE relation_factor alter_table_actions - | ALTER TABLE relation_factor alter_column_group_option - | ALTER EXTERNAL TABLE relation_factor ADD PARTITION LeftParen add_external_table_partition_actions RightParen LOCATION STRING_VALUE - | ALTER EXTERNAL TABLE relation_factor DROP PARTITION LOCATION STRING_VALUE + | ALTER TABLE relation_factor alter_column_group_action + | ALTER EXTERNAL TABLE relation_factor alter_external_table_action + ; + +alter_external_table_action + : ADD PARTITION LeftParen add_external_table_partition_actions? RightParen LOCATION STRING_VALUE + | DROP PARTITION LOCATION STRING_VALUE ; add_external_table_partition_actions : add_external_table_partition_action - | empty | add_external_table_partition_actions Comma add_external_table_partition_action ; @@ -3815,7 +3819,7 @@ visibility_option | INVISIBLE ; -alter_column_group_option +alter_column_group_action : (ADD|DROP) COLUMN GROUP LeftParen column_group_list RightParen ; @@ -4579,7 +4583,11 @@ date_unit_for_extract ; json_mergepatch_expr - : JSON_MERGEPATCH LeftParen bit_expr Comma bit_expr js_mp_return_clause? opt_json_mergepatch json_mergepatch_on_error? RightParen + : JSON_MERGEPATCH LeftParen bit_expr Comma bit_expr js_mp_return_clause? json_mergepatch_opt RightParen + ; + +json_mergepatch_opt + : opt_json_mergepatch json_mergepatch_on_error? ; json_mergepatch_on_error @@ -4623,7 +4631,11 @@ js_array_return_clause ; json_value_expr - : JSON_VALUE LeftParen js_doc_expr Comma js_literal opt_js_value_returning_type TRUNCATE? ASCII? json_value_on_opt? RightParen + : JSON_VALUE LeftParen js_doc_expr Comma js_literal opt_js_value_returning_type json_value_opt RightParen + ; + +json_value_opt + : TRUNCATE? ASCII? json_value_on_opt? ; json_equal_expr @@ -4735,7 +4747,11 @@ json_exists_response_type ; json_query_expr - : JSON_QUERY LeftParen js_doc_expr Comma js_literal (RETURNING js_query_return_type)? TRUNCATE? scalars_opt? PRETTY? ASCII? wrapper_opts? ASIS? json_query_on_opt? MULTIVALUE? RightParen + : JSON_QUERY LeftParen js_doc_expr Comma js_literal (RETURNING js_query_return_type)? json_query_opt RightParen + ; + +json_query_opt + : TRUNCATE? scalars_opt? PRETTY? ASCII? wrapper_opts? ASIS? json_query_on_opt? MULTIVALUE? ; json_query_on_opt diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableActionFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableActionFactoryTest.java index 08c8c26752..0684519be0 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableActionFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableActionFactoryTest.java @@ -35,6 +35,7 @@ import com.oceanbase.tools.sqlparser.statement.common.CharacterType; import com.oceanbase.tools.sqlparser.statement.common.GeneralDataType; import com.oceanbase.tools.sqlparser.statement.common.RelationFactor; +import com.oceanbase.tools.sqlparser.statement.common.mysql.LobStorageOption; import com.oceanbase.tools.sqlparser.statement.createtable.ColumnDefinition; import com.oceanbase.tools.sqlparser.statement.createtable.ConstraintState; import com.oceanbase.tools.sqlparser.statement.createtable.HashPartition; @@ -147,6 +148,23 @@ public void generate_addColumns_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_addColumnsWithLobStorage_succeed() { + StatementFactory factory = new MySQLAlterTableActionFactory( + getActionContext("add (id varchar(64), id1 blob) json(col) store as (chunk 'aas' chunk 123)")); + AlterTableAction actual = factory.generate(); + + AlterTableAction expect = new AlterTableAction(); + CharacterType t1 = new CharacterType("varchar", new BigDecimal("64")); + ColumnDefinition d1 = new ColumnDefinition(new ColumnReference(null, null, "id"), t1); + GeneralDataType t2 = new GeneralDataType("blob", null); + ColumnDefinition d2 = new ColumnDefinition(new ColumnReference(null, null, "id1"), t2); + expect.setAddColumns(Arrays.asList(d1, d2)); + LobStorageOption storageOption = new LobStorageOption("col", Arrays.asList("'aas'", "123")); + expect.setLobStorageOption(storageOption); + Assert.assertEquals(expect, actual); + } + @Test public void generate_dropColumnCascade_succeed() { StatementFactory factory = new MySQLAlterTableActionFactory( @@ -208,6 +226,19 @@ public void generate_alterColumn_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_alterColumn1_succeed() { + StatementFactory factory = new MySQLAlterTableActionFactory( + getActionContext("alter column a.b set default (12)")); + AlterTableAction actual = factory.generate(); + + AlterTableAction expect = new AlterTableAction(); + AlterColumnBehavior behavior = new AlterColumnBehavior(); + behavior.setDefaultValue(new ConstExpression("12")); + expect.alterColumnBehavior(new ColumnReference(null, "a", "b"), behavior); + Assert.assertEquals(expect, actual); + } + @Test public void generate_alterColumnDropDefault_succeed() { StatementFactory factory = new MySQLAlterTableActionFactory( @@ -642,6 +673,17 @@ public void generate_dropForeignConstraints_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_exchangePartition_succeed() { + StatementFactory factory = new MySQLAlterTableActionFactory( + getActionContext("exchange partition p1 with table tbl without validation")); + AlterTableAction actual = factory.generate(); + + AlterTableAction expect = new AlterTableAction(); + expect.setExchangePartition("p1", new RelationFactor("tbl")); + Assert.assertEquals(expect, actual); + } + @Test public void generate_dropPrimaryKey_succeed() { StatementFactory factory = new MySQLAlterTableActionFactory( diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableFactoryTest.java index 7cce672361..cff06c4a86 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableFactoryTest.java @@ -18,6 +18,8 @@ import java.math.BigDecimal; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -29,6 +31,7 @@ import com.oceanbase.tools.sqlparser.obmysql.OBLexer; import com.oceanbase.tools.sqlparser.obmysql.OBParser; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_table_stmtContext; +import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTable; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTableAction; import com.oceanbase.tools.sqlparser.statement.common.CharacterType; @@ -37,6 +40,7 @@ import com.oceanbase.tools.sqlparser.statement.createtable.ColumnDefinition; import com.oceanbase.tools.sqlparser.statement.createtable.TableOptions; import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; public class MySQLAlterTableFactoryTest { @@ -71,6 +75,52 @@ public void generate_alterTable1_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_alterExternalTableAddEmptyPartition_succeed() { + StatementFactory factory = new MySQLAlterTableFactory( + getAlterContext("alter external table a.b add partition () location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + action.setAddExternalTablePartition(new HashMap<>()); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_alterExternalTableAddPartition_succeed() { + StatementFactory factory = new MySQLAlterTableFactory( + getAlterContext( + "alter external table a.b add partition (col='aaa', col1=@@global.ss) location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + Map partitions = new HashMap<>(); + partitions.put("col", new ConstExpression("'aaa'")); + partitions.put("col1", new ConstExpression("@@global.ss")); + action.setAddExternalTablePartition(partitions); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_alterExternalTableDropPartition_succeed() { + StatementFactory factory = new MySQLAlterTableFactory( + getAlterContext("alter external table a.b drop partition location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + action.setDropExternalTablePartition(true); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + @Test public void generate_alterExternalTable1_succeed() { StatementFactory factory = new MySQLAlterTableFactory( @@ -121,4 +171,5 @@ private RelationFactor getRelationFactor(String schema, String relation) { relationFactor.setSchema(schema); return relationFactor; } + } diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateIndexFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateIndexFactoryTest.java index 7fddcceb7e..7317b280cd 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateIndexFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateIndexFactoryTest.java @@ -15,9 +15,7 @@ */ package com.oceanbase.tools.sqlparser.adapter; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -194,13 +192,20 @@ public void generate_withColumnGroup_customGroup_succeed() { @Test public void generate_CreateVectorIndex_Succeed() { StatementFactory factory = new MySQLCreateIndexFactory( - getCreateIdxContext("create vector index vec_idx1 on t_vec(c2) with (distance=l2, type=hnsw);")); + getCreateIdxContext("create vector index vec_idx1 on t_vec(c2) " + + "key_block_size=1234 with (distance=l2, type=hnsw);")); CreateIndex actual = factory.generate(); CreateIndex expected = new CreateIndex(new RelationFactor("vec_idx1"), - new RelationFactor("t_vec"), Arrays.asList( + new RelationFactor("t_vec"), Collections.singletonList( new SortColumn(new ColumnReference(null, null, "c2")))); - expected.setIndexOptions(new IndexOptions()); + IndexOptions indexOptions = new IndexOptions(); + indexOptions.setKeyBlockSize(1234); + Map params = new HashMap<>(); + params.put("distance", "l2"); + params.put("type", "hnsw"); + indexOptions.setVectorIndexParams(params); + expected.setIndexOptions(indexOptions); expected.setVector(true); Assert.assertEquals(expected, actual); diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateTableFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateTableFactoryTest.java index 7ff458254f..0d203c3201 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateTableFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateTableFactoryTest.java @@ -39,6 +39,7 @@ import com.oceanbase.tools.sqlparser.statement.common.ColumnGroupElement; import com.oceanbase.tools.sqlparser.statement.common.DataType; import com.oceanbase.tools.sqlparser.statement.common.RelationFactor; +import com.oceanbase.tools.sqlparser.statement.common.mysql.LobStorageOption; import com.oceanbase.tools.sqlparser.statement.common.mysql.VectorType; import com.oceanbase.tools.sqlparser.statement.createtable.ColumnAttributes; import com.oceanbase.tools.sqlparser.statement.createtable.ColumnDefinition; @@ -143,6 +144,23 @@ public void generate_createTableAsSelect_generateSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_createTableAsSelectIgnore_generateSucceed() { + Create_table_stmtContext context = + getCreateTableContext("create table any_schema.abcd ignore as select * from tab"); + StatementFactory factory = new MySQLCreateTableFactory(context); + CreateTable actual = factory.generate(); + + CreateTable expect = new CreateTable(context, getRelationFactor("any_schema", "abcd")); + expect.setIgnore(true); + NameReference from = new NameReference(null, "tab", null); + SelectBody selectBody = + new SelectBody(Collections.singletonList(new Projection()), Collections.singletonList(from)); + Select select = new Select(selectBody); + expect.setAs(select); + Assert.assertEquals(expect, actual); + } + @Test public void generate_sortKey_succeed() { Create_table_stmtContext context = @@ -365,18 +383,60 @@ public void generate_physicalAttrs_succeed() { @Test public void generate_formatTableOp_succeed() { - Create_table_stmtContext context = getCreateTableContext( - "create table any_schema.abcd (id varchar(64)) kv_attributes='12' format=(ENCODING='aaaa',LINE_DELIMITER=123," - + "SKIP_HEADER=12," - + "EMPTY_FIELD_AS_NULL=true,NULL_IF_EXETERNAL=(1,2,3))"); + Create_table_stmtContext context = getCreateTableContext("create table any_schema.abcd (id varchar(64)) " + + "kv_attributes='12' " + + "key_block_size=456 " + + "AUTO_INCREMENT_CACHE_SIZE=667 " + + "partition_type=USER_SPECIFIED " + + "properties=(ACCESSID='abcd', PROJECT_NAME='ooop') " + + "json(col) store as (chunk 'aas' chunk 123) " + + "micro_index_clustered=true " + + "auto_refresh=OFF " + + "max_rows=123445 " + + "min_rows=9879 " + + "password='asdasd' " + + "pack_keys=default " + + "connection='asasdasd' " + + "data directory='ooopi' " + + "index directory='indxasdasd' " + + "encryption='asdasdasdas' " + + "stats_auto_recalc=default " + + "stats_persistent=default " + + "stats_sample_pages=3345 " + + "union=(col1, col2) " + + "insert_method=FIRST " + + "format=(ENCODING='aaaa',LINE_DELIMITER=123,SKIP_HEADER=12,EMPTY_FIELD_AS_NULL=true," + + "NULL_IF_EXETERNAL=(1,2,3),COMPRESSION=col3)"); StatementFactory factory = new MySQLCreateTableFactory(context); CreateTable actual = factory.generate(); - CreateTable expect = new CreateTable(context, getRelationFactor("any_schema", "abcd")); DataType dataType = new CharacterType("varchar", new BigDecimal("64")); expect.setTableElements( Collections.singletonList(new ColumnDefinition(new ColumnReference(null, null, "id"), dataType))); TableOptions tableOptions = new TableOptions(); + tableOptions.setAutoIncrementCacheSize(667); + tableOptions.setDataDirectory("'ooopi'"); + tableOptions.setIndexDirectory("'indxasdasd'"); + tableOptions.setStatsAutoRecalc("default"); + tableOptions.setStatsPersistent("default"); + tableOptions.setStatsSamplePages("3345"); + tableOptions.setUnion(Arrays.asList(new RelationFactor("col1"), new RelationFactor("col2"))); + tableOptions.setInsertMethod("FIRST"); + tableOptions.setEncryption("'asdasdasdas'"); + tableOptions.setPartitionType("USER_SPECIFIED"); + Map externalProperties = new HashMap<>(); + externalProperties.put("ACCESSID", "'abcd'"); + externalProperties.put("PROJECT_NAME", "'ooop'"); + tableOptions.setExternalProperties(externalProperties); + LobStorageOption storageOption = new LobStorageOption("col", Arrays.asList("'aas'", "123")); + tableOptions.setMicroIndexClustered(true); + tableOptions.setLobStorageOption(storageOption); + tableOptions.setAutoRefresh("OFF"); + tableOptions.setMaxRows(123445); + tableOptions.setMinRows(9879); + tableOptions.setPassword("'asdasd'"); + tableOptions.setPackKeys("default"); + tableOptions.setConnection("'asasdasd'"); Map map = new HashMap<>(); map.put("ENCODING", new ConstExpression("'aaaa'")); map.put("EMPTY_FIELD_AS_NULL", new BoolValue(true)); @@ -386,8 +446,13 @@ public void generate_formatTableOp_succeed() { es.addExpression(new ConstExpression("2")); es.addExpression(new ConstExpression("3")); map.put("NULL_IF_EXETERNAL", es); + map.put("COMPRESSION", new ConstExpression("col3")); map.put("LINE_DELIMITER", new ConstExpression("123")); + tableOptions.setKeyBlockSize(456); + tableOptions.setFormat(map); + tableOptions.setKeyBlockSize(456); + tableOptions.setAutoIncrementCacheSize(667); tableOptions.setKvAttributes("'12'"); expect.setTableOptions(tableOptions); Assert.assertEquals(expect, actual); @@ -507,7 +572,12 @@ public void generate_WithVectorIndex_Succeed() { SortColumn indexColumn = new SortColumn(new ColumnReference(null, null, "c1")); OutOfLineIndex index = new OutOfLineIndex("idx1", Collections.singletonList(indexColumn)); index.setVector(true); - index.setIndexOptions(new IndexOptions()); + IndexOptions indexOptions = new IndexOptions(); + index.setIndexOptions(indexOptions); + Map params = new HashMap<>(); + params.put("distance", "L2"); + params.put("type", "hnsw"); + indexOptions.setVectorIndexParams(params); expect.setTableElements( Arrays.asList(new ColumnDefinition(new ColumnReference(null, null, "c1"), dataType), index)); Assert.assertEquals(expect, actual); @@ -534,4 +604,5 @@ private RelationFactor getRelationFactor(String schema, String relation) { relationFactor.setSchema(schema); return relationFactor; } + } diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLExpressionFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLExpressionFactoryTest.java index 3f875de38c..16c299ccf2 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLExpressionFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLExpressionFactoryTest.java @@ -43,23 +43,7 @@ import com.oceanbase.tools.sqlparser.statement.common.WindowOffsetType; import com.oceanbase.tools.sqlparser.statement.common.WindowSpec; import com.oceanbase.tools.sqlparser.statement.common.WindowType; -import com.oceanbase.tools.sqlparser.statement.expression.ArrayExpression; -import com.oceanbase.tools.sqlparser.statement.expression.BoolValue; -import com.oceanbase.tools.sqlparser.statement.expression.CaseWhen; -import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; -import com.oceanbase.tools.sqlparser.statement.expression.CompoundExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ExpressionParam; -import com.oceanbase.tools.sqlparser.statement.expression.FullTextSearch; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionParam; -import com.oceanbase.tools.sqlparser.statement.expression.GroupConcat; -import com.oceanbase.tools.sqlparser.statement.expression.IntervalExpression; -import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption; -import com.oceanbase.tools.sqlparser.statement.expression.NullExpression; -import com.oceanbase.tools.sqlparser.statement.expression.TextSearchMode; -import com.oceanbase.tools.sqlparser.statement.expression.WhenClause; +import com.oceanbase.tools.sqlparser.statement.expression.*; import com.oceanbase.tools.sqlparser.statement.select.OrderBy; import com.oceanbase.tools.sqlparser.statement.select.SortDirection; import com.oceanbase.tools.sqlparser.statement.select.SortKey; @@ -222,6 +206,23 @@ public void generate_matchNaturalMode_generateSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_matchNaturalModeWithQueryExpansion_generateSucceed() { + ExprContext context = getExprContext( + "match(col,tab.col,chz.tab.col) against ('abc' IN NATURAL LANGUAGE MODE WITH QUERY EXPANSION)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ColumnReference(null, null, "col"))); + params.add(new ExpressionParam(new ColumnReference(null, "tab", "col"))); + params.add(new ExpressionParam(new ColumnReference("chz", "tab", "col"))); + FullTextSearch expect = new FullTextSearch(params, "'abc'"); + expect.setWithQueryExpansion(true); + expect.setSearchMode(TextSearchMode.NATURAL_LANGUAGE_MODE); + Assert.assertEquals(expect, actual); + } + @Test public void generate_matchBooleanMode_generateSucceed() { ExprContext context = getExprContext("match(col,tab.col,chz.tab.col) against ('abc' IN BOOLEAN MODE)"); @@ -338,6 +339,28 @@ public void generate_st_asmvt_generateFunctionCallSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_lastRefreshScn_generateFunctionCallSucceed() { + ExprContext context = getExprContext("last_refresh_scn(123)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + FunctionCall expect = new FunctionCall("last_refresh_scn", + Collections.singletonList(new ExpressionParam(new ConstExpression("123")))); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_sumOpnsize_generateFunctionCallSucceed() { + ExprContext context = getExprContext("sum_Opnsize(123)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + FunctionCall expect = new FunctionCall("sum_Opnsize", + Collections.singletonList(new ExpressionParam(new ConstExpression("123")))); + Assert.assertEquals(expect, actual); + } + @Test public void generate_stddevPopExpr_generateFunctionCallSucceed() { ExprContext context = getExprContext("STDDEV_POP(all 1)"); @@ -453,7 +476,7 @@ public void generate_groupConcat_generateFunctionCallSucceed() { @Test public void generate_castAsChar_generateFunctionCallSucceed() { - ExprContext context = getExprContext("cast('abc' as character(15) binary)"); + ExprContext context = getExprContext("cast('abc' as character(15) binary array)"); StatementFactory factory = new MySQLExpressionFactory(context); Expression actual = factory.generate(); @@ -462,6 +485,7 @@ public void generate_castAsChar_generateFunctionCallSucceed() { CharacterType type = new CharacterType("character", new BigDecimal(15)); type.setBinary(true); p.addOption(type); + p.addOption(new ConstExpression("array")); params.add(p); FunctionCall expect = new FunctionCall("cast", params); Assert.assertEquals(expect, actual); @@ -756,6 +780,258 @@ public void generate_weightString3Int_generateFunctionCallSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_jsonQueryExpr_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc')"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new JsonOption()); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExprFullOpts_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "TRUNCATE " + + "allow scalars " + + "pretty " + + "ASCII " + + "with unconditional array wrapper " + + "asis " + + "empty on empty empty array on error_p error_p on mismatch " + + "MULTIVALUE)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setScalarsMode(JsonOption.ScalarsMode.ALLOW_SCALARS); + jsonOpt.setPretty(true); + jsonOpt.setAscii(true); + jsonOpt.setMultiValue(true); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_UNCONDITIONAL_ARRAY_WRAPPER); + jsonOpt.setAsis(true); + JsonOnOption jsonOnOption = new JsonOnOption(); + jsonOnOption.setOnEmpty(new ConstExpression("empty")); + jsonOnOption.setOnError(new ConstExpression("empty array")); + jsonOnOption.setOnMismatches(Collections.singletonList( + new JsonOnOption.OnMismatch(new ConstExpression("error_p"), null))); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExprFullOpts2_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "empty array on empty empty array on error_p error_p on mismatch)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + JsonOnOption jsonOnOption = new JsonOnOption(); + jsonOnOption.setOnEmpty(new ConstExpression("empty array")); + jsonOnOption.setOnError(new ConstExpression("empty array")); + jsonOnOption.setOnMismatches(Collections.singletonList( + new JsonOnOption.OnMismatch(new ConstExpression("error_p"), null))); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExprFullOpts4_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "empty object on empty empty array on error_p error_p on mismatch)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + JsonOnOption jsonOnOption = new JsonOnOption(); + jsonOnOption.setOnEmpty(new ConstExpression("empty object")); + jsonOnOption.setOnError(new ConstExpression("empty array")); + jsonOnOption.setOnMismatches(Collections.singletonList( + new JsonOnOption.OnMismatch(new ConstExpression("error_p"), null))); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr1_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "with array wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_ARRAY_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr2_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "with conditional wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_CONDITIONAL_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr3_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "with unconditional wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_UNCONDITIONAL_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr4_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "with wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr5_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "without wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITHOUT_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr6_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "without array wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITHOUT_ARRAY_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExprFullOpts1_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "TRUNCATE " + + "disallow scalars " + + "pretty " + + "ASCII " + + "with conditional array wrapper " + + "asis " + + "error_p on empty empty object on error_p dot on mismatch " + + "MULTIVALUE)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setScalarsMode(JsonOption.ScalarsMode.DISALLOW_SCALARS); + jsonOpt.setPretty(true); + jsonOpt.setAscii(true); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_CONDITIONAL_ARRAY_WRAPPER); + jsonOpt.setAsis(true); + jsonOpt.setMultiValue(true); + JsonOnOption jsonOnOption = new JsonOnOption(); + jsonOnOption.setOnEmpty(new ConstExpression("error_p")); + jsonOnOption.setOnError(new ConstExpression("empty object")); + jsonOnOption.setOnMismatches(Collections.singletonList( + new JsonOnOption.OnMismatch(new ConstExpression("dot"), null))); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + @Test public void generate_jsonValueExpr_generateFunctionCallSucceed() { ExprContext context = getExprContext("JSON_VALUE('123', _utf8 'abc' returning double TRUNCATE ASCII)"); @@ -768,8 +1044,10 @@ public void generate_jsonValueExpr_generateFunctionCallSucceed() { params.add(p); FunctionCall expect = new FunctionCall("JSON_VALUE", params); expect.addOption(new NumberType("double", null, null)); - expect.addOption(new ConstExpression("TRUNCATE")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -786,11 +1064,13 @@ public void generate_jsonValueExprOnEmpty_generateFunctionCallSucceed() { params.add(p); FunctionCall expect = new FunctionCall("JSON_VALUE", params); expect.addOption(new NumberType("double", null, null)); - expect.addOption(new ConstExpression("TRUNCATE")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnEmpty(new ConstExpression("error_p")); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -807,11 +1087,13 @@ public void generate_jsonValueExprOnError_generateFunctionCallSucceed() { params.add(p); FunctionCall expect = new FunctionCall("JSON_VALUE", params); expect.addOption(new NumberType("double", null, null)); - expect.addOption(new ConstExpression("TRUNCATE")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new NullExpression()); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -828,12 +1110,30 @@ public void generate_jsonValueExprOnErrorEmpty_generateFunctionCallSucceed() { params.add(p); FunctionCall expect = new FunctionCall("JSON_VALUE", params); expect.addOption(new NumberType("double", null, null)); - expect.addOption(new ConstExpression("TRUNCATE")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new NullExpression()); jsonOnOption.setOnEmpty(new ConstExpression("12")); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonValueExprNoOpt_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_VALUE('123', _utf8 'abc' returning double)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + FunctionParam p = new ExpressionParam(new ConstExpression("_utf8 'abc'")); + params.add(p); + FunctionCall expect = new FunctionCall("JSON_VALUE", params); + expect.addOption(new NumberType("double", null, null)); + expect.addOption(new JsonOption()); Assert.assertEquals(expect, actual); } @@ -1430,6 +1730,20 @@ public void generate_vectorDistanceExpr_Succeed() { Assert.assertEquals(expected, actual); } + @Test + public void generate_vectorDistanceExpr1_Succeed() { + ExprContext context = getExprContext("VECTOR_DISTANCE(vector1, vector2, COSINE)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ColumnReference(null, null, "vector1"))); + params.add(new ExpressionParam(new ColumnReference(null, null, "vector2"))); + params.add(new ExpressionParam(new ConstExpression("COSINE"))); + FunctionCall expected = new FunctionCall("VECTOR_DISTANCE", params); + Assert.assertEquals(expected, actual); + } + @Test public void generate_AnyArrayExpr_1_Succeed() { ExprContext context = getExprContext("\"hel\" = ANY([\"hello\", \"hi\"])"); diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLFromReferenceFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLFromReferenceFactoryTest.java index c840f8744d..62d9da22a0 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLFromReferenceFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLFromReferenceFactoryTest.java @@ -15,10 +15,7 @@ */ package com.oceanbase.tools.sqlparser.adapter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -38,14 +35,7 @@ import com.oceanbase.tools.sqlparser.statement.common.BraceBlock; import com.oceanbase.tools.sqlparser.statement.common.GeneralDataType; import com.oceanbase.tools.sqlparser.statement.common.NumberType; -import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; -import com.oceanbase.tools.sqlparser.statement.expression.CompoundExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ExpressionParam; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionParam; -import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption; -import com.oceanbase.tools.sqlparser.statement.expression.NullExpression; +import com.oceanbase.tools.sqlparser.statement.expression.*; import com.oceanbase.tools.sqlparser.statement.select.ExpressionReference; import com.oceanbase.tools.sqlparser.statement.select.FlashBackType; import com.oceanbase.tools.sqlparser.statement.select.FlashbackUsage; @@ -169,6 +159,7 @@ public void generate_jsonTable2_generateSucceed() { FunctionParam op2 = new ExpressionParam(new ColumnReference(null, null, "col1")); op2.addOption(new NumberType("int", null, null)); op2.addOption(new ConstExpression("123")); + op2.addOption(new JsonOption()); fcall.addOption(op2); FunctionParam op3 = new ExpressionParam(new ColumnReference(null, null, "col2")); @@ -177,7 +168,9 @@ public void generate_jsonTable2_generateSucceed() { op3.addOption(new ConstExpression("123")); JsonOnOption onOption = new JsonOnOption(); onOption.setOnEmpty(new NullExpression()); - op3.addOption(onOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(onOption); + op3.addOption(jsonOpt); fcall.addOption(op3); FunctionParam op4 = new ExpressionParam(new ColumnReference(null, null, "col3")); @@ -186,7 +179,9 @@ public void generate_jsonTable2_generateSucceed() { op4.addOption(new ConstExpression("123")); onOption = new JsonOnOption(); onOption.setOnError(new ConstExpression("error_p")); - op4.addOption(onOption); + jsonOpt = new JsonOption(); + jsonOpt.setOnOption(onOption); + op4.addOption(jsonOpt); fcall.addOption(op4); FunctionParam op5 = new ExpressionParam(new ColumnReference(null, null, "col4")); @@ -196,7 +191,9 @@ public void generate_jsonTable2_generateSucceed() { onOption = new JsonOnOption(); onOption.setOnEmpty(new NullExpression()); onOption.setOnError(new ConstExpression("error_p")); - op5.addOption(onOption); + jsonOpt = new JsonOption(); + jsonOpt.setOnOption(onOption); + op5.addOption(jsonOpt); fcall.addOption(op5); ExpressionReference expect = new ExpressionReference(fcall, null); Assert.assertEquals(expect, actual); @@ -298,6 +295,21 @@ public void generate_nameRefWithPartition_generateNameRefSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_nameRefWithExternalPartition_generateNameRefSucceed() { + Table_referenceContext context = getTableReferenceContext( + "select a from mysql.tab partition (col1=@@global.lalal, col2='aaa')"); + StatementFactory factory = new MySQLFromReferenceFactory(context); + FromReference actual = factory.generate(); + + NameReference expect = new NameReference("mysql", "tab", null); + Map partitionMap = new HashMap<>(); + partitionMap.put("col1", new ConstExpression("@@global.lalal")); + partitionMap.put("col2", new ConstExpression("'aaa'")); + expect.setPartitionUsage(new PartitionUsage(PartitionType.PARTITION, partitionMap)); + Assert.assertEquals(expect, actual); + } + @Test public void generate_nameRefWithFlashback_generateNameRefSucceed() { Table_referenceContext context = diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLInsertFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLInsertFactoryTest.java index 5d58c3fe27..e0f89ab1f6 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLInsertFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLInsertFactoryTest.java @@ -71,6 +71,24 @@ public void generate_simpleInsert_succeed() { Assert.assertEquals(actual, expect); } + @Test + public void generate_insertWithHighPriority_succeed() { + StatementFactory factory = new MySQLInsertFactory( + getInsertContext("insert high_priority overwrite a.b values(1,default)")); + Insert actual = factory.generate(); + + RelationFactor factor = new RelationFactor("b"); + factor.setSchema("a"); + InsertTable insertTable = new InsertTable(factor); + List> values = new ArrayList<>(); + values.add(Arrays.asList(new ConstExpression("1"), new ConstExpression("default"))); + insertTable.setValues(values); + Insert expect = new Insert(Collections.singletonList(insertTable), null); + expect.setHighPriority(true); + expect.setOverwrite(true); + Assert.assertEquals(actual, expect); + } + @Test public void generate_simpleReplace_succeed() { StatementFactory factory = new MySQLInsertFactory(getInsertContext( diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLSelectFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLSelectFactoryTest.java index f5ee27b5cc..3ac36da245 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLSelectFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLSelectFactoryTest.java @@ -116,7 +116,7 @@ public void generate_parentSelectUnionDual_generateSelectSucceed() { @Test public void generate_orderAndLimitUnion_generateSelectSucceed() { Select_stmtContext context = getSelectContext( - "select col.* abc from tab order by col desc limit 3 union distinct select * from dual"); + "select col.* abc from tab order by col desc approx limit 3 union distinct select * from dual"); StatementFactory factory = new MySQLSelectFactory(context); Select actual = factory.generate(); @@ -249,6 +251,7 @@ public void generate_fromSelectStatment_generateSelectSucceed() { fromBody.setOrderBy(orderBy); FlashbackUsage flashbackUsage = new FlashbackUsage(FlashBackType.AS_OF_SNAPSHOT, new ConstExpression("1")); ExpressionReference from = new ExpressionReference(fromBody, "abc"); + from.setLateral(true); from.setFlashbackUsage(flashbackUsage); Select expect = new Select(new SelectBody(Collections.singletonList(p), Collections.singletonList(from))); Assert.assertEquals(expect, actual); diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java index d11b8a92e8..aae65c06fd 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java @@ -16,10 +16,7 @@ package com.oceanbase.tools.sqlparser.adapter; import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -86,6 +83,36 @@ public void generate_columnDef_generateSuccees() { Assert.assertEquals(expect, actual); } + @Test + public void generate_serialColumnDef_generateSuccees() { + StatementFactory factory = + new MySQLTableElementFactory(getTableElementContext("tb.col serial")); + ColumnDefinition actual = (ColumnDefinition) factory.generate(); + + ColumnDefinition expect = new ColumnDefinition(new ColumnReference(null, "tb", "col"), null); + expect.setSerial(true); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_columnDefWithForeignKeyIndexName_succeed() { + StatementFactory factory = new MySQLTableElementFactory(getTableElementContext( + "tb.col varchar(64) references a.b (col2, col3) match simple on delete cascade on update set default")); + TableElement actual = factory.generate(); + + DataType dataType = new CharacterType("varchar", new BigDecimal("64")); + ColumnDefinition expect = new ColumnDefinition(new ColumnReference(null, "tb", "col"), dataType); + + ColumnReference r1 = new ColumnReference(null, null, "col2"); + ColumnReference r2 = new ColumnReference(null, null, "col3"); + ForeignReference reference = new ForeignReference("a", "b", Arrays.asList(r1, r2)); + reference.setDeleteOption(OnOption.CASCADE); + reference.setUpdateOption(OnOption.SET_DEFAULT); + reference.setMatchOption(MatchOption.SIMPLE); + expect.setForeignReference(reference); + Assert.assertEquals(expect, actual); + } + @Test public void generate_columnDefFirst_generateSuccees() { StatementFactory factory = @@ -200,10 +227,25 @@ public void generate_columnDefDefaultExpr_generateSuccees() { Assert.assertEquals(expect, actual); } + @Test + public void generate_columnDefDefaultExpr1_generateSuccees() { + StatementFactory factory = + new MySQLTableElementFactory(getTableElementContext("tb.col varchar(64) default (1) chunk '123'")); + ColumnDefinition actual = (ColumnDefinition) factory.generate(); + + DataType dataType = new CharacterType("varchar", new BigDecimal("64")); + ColumnDefinition expect = new ColumnDefinition(new ColumnReference(null, "tb", "col"), dataType); + ColumnAttributes attributes = new ColumnAttributes(); + attributes.setDefaultValue(new ConstExpression("1")); + attributes.setLobChunkSize("'123'"); + expect.setColumnAttributes(attributes); + Assert.assertEquals(expect, actual); + } + @Test public void generate_columnDefKey_generateSuccees() { StatementFactory factory = - new MySQLTableElementFactory(getTableElementContext("tb.col varchar(64) key")); + new MySQLTableElementFactory(getTableElementContext("tb.col varchar(64) key chunk 123")); ColumnDefinition actual = (ColumnDefinition) factory.generate(); DataType dataType = new CharacterType("varchar", new BigDecimal("64")); @@ -212,6 +254,7 @@ public void generate_columnDefKey_generateSuccees() { InLineConstraint constraint = new InLineConstraint(null, null); constraint.setPrimaryKey(true); attributes.setConstraints(Collections.singletonList(constraint)); + attributes.setLobChunkSize("123"); expect.setColumnAttributes(attributes); Assert.assertEquals(expect, actual); } @@ -219,7 +262,8 @@ public void generate_columnDefKey_generateSuccees() { @Test public void generate_columnDefOrigDefaultExpr_generateSuccees() { StatementFactory factory = new MySQLTableElementFactory( - getTableElementContext("tb.col varchar(64) orig_default current_timestamp(1)")); + getTableElementContext( + "tb.col varchar(64) orig_default current_timestamp(1) column_format default storage disk")); ColumnDefinition actual = (ColumnDefinition) factory.generate(); DataType dataType = new CharacterType("varchar", new BigDecimal("64")); @@ -228,6 +272,8 @@ public void generate_columnDefOrigDefaultExpr_generateSuccees() { FunctionCall expr = new FunctionCall("current_timestamp", Collections.singletonList(new ExpressionParam(new ConstExpression("1")))); attributes.setOrigDefault(expr); + attributes.setColumnFormat("default"); + attributes.setStorage("disk"); expect.setColumnAttributes(attributes); Assert.assertEquals(expect, actual); } @@ -1118,8 +1164,13 @@ public void generate_outOfLineIndexVectorIndex_succeed() { TableElement actual = factory.generate(); OutOfLineIndex expected = new OutOfLineIndex("idx1", Collections.singletonList(new SortColumn(new ColumnReference(null, null, "c2")))); - expected.setIndexOptions(new IndexOptions()); + IndexOptions indexOptions = new IndexOptions(); + expected.setIndexOptions(indexOptions); expected.setVector(true); + Map params = new HashMap<>(); + params.put("distance", "L2"); + params.put("type", "hnsw"); + indexOptions.setVectorIndexParams(params); Assert.assertEquals(expected, actual); } diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableActionFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableActionFactoryTest.java index 94415e83de..c82d947b05 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableActionFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableActionFactoryTest.java @@ -82,6 +82,17 @@ public void generate_tableOptions_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_exchangePartition_succeed() { + StatementFactory factory = new OracleAlterTableActionFactory( + getActionContext("exchange partition p1 with table tbl including indexes without validation")); + AlterTableAction actual = factory.generate(); + + AlterTableAction expect = new AlterTableAction(); + expect.setExchangePartition("p1", new RelationFactor("tbl")); + Assert.assertEquals(expect, actual); + } + @Test public void generate_setInterval_succeed() { StatementFactory factory = new OracleAlterTableActionFactory( diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableFactoryTest.java index 95b8339865..65b33773b9 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableFactoryTest.java @@ -18,6 +18,8 @@ import java.math.BigDecimal; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -29,6 +31,7 @@ import com.oceanbase.tools.sqlparser.oboracle.OBLexer; import com.oceanbase.tools.sqlparser.oboracle.OBParser; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_table_stmtContext; +import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTable; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTableAction; import com.oceanbase.tools.sqlparser.statement.common.CharacterType; @@ -37,6 +40,7 @@ import com.oceanbase.tools.sqlparser.statement.createtable.ColumnDefinition; import com.oceanbase.tools.sqlparser.statement.createtable.TableOptions; import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; /** * Test cases for {@link OracleAlterTableFactory} @@ -100,6 +104,52 @@ public void generate_alterTable_dropColumnGroup_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_alterExternalTableAddEmptyPartition_succeed() { + StatementFactory factory = new OracleAlterTableFactory( + getAlterContext("alter external table a.b add partition () location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + action.setAddExternalTablePartition(new HashMap<>()); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_alterExternalTableAddPartition_succeed() { + StatementFactory factory = new OracleAlterTableFactory( + getAlterContext( + "alter external table a.b add partition (col='aaa', col1=@@global.ss) location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + Map partitions = new HashMap<>(); + partitions.put("col", new ConstExpression("'aaa'")); + partitions.put("col1", new ConstExpression("@@global.ss")); + action.setAddExternalTablePartition(partitions); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_alterExternalTableDropPartition_succeed() { + StatementFactory factory = new OracleAlterTableFactory( + getAlterContext("alter external table a.b drop partition location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + action.setDropExternalTablePartition(true); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + private Alter_table_stmtContext getAlterContext(String action) { OBLexer lexer = new OBLexer(CharStreams.fromString(action)); CommonTokenStream tokens = new CommonTokenStream(lexer); @@ -113,4 +163,5 @@ private RelationFactor getRelationFactor(String schema, String relation) { relationFactor.setSchema(schema); return relationFactor; } + } diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateIndexFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateIndexFactoryTest.java index 4b6ccbb093..b9bd719255 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateIndexFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateIndexFactoryTest.java @@ -59,6 +59,20 @@ public void generate_createIndex_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_createMdSysIndex_succeed() { + StatementFactory factory = new OracleCreateIndexFactory( + getCreateIdxContext("create index abc on tb (col, col1) indextype is mdsys.spatial_index ")); + CreateIndex actual = factory.generate(); + + CreateIndex expect = new CreateIndex(new RelationFactor("abc"), + new RelationFactor("tb"), Arrays.asList( + new SortColumn(new RelationReference("col", null)), + new SortColumn(new RelationReference("col1", null)))); + expect.setMdSysDotSpatialIndex(true); + Assert.assertEquals(expect, actual); + } + @Test public void generate_createUniqueIndex_succeed() { StatementFactory factory = new OracleCreateIndexFactory( diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateTableFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateTableFactoryTest.java index 6277c73730..a4ab213ef0 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateTableFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateTableFactoryTest.java @@ -358,9 +358,13 @@ public void generate_tableWithPartition_succeed() { @Test public void generate_formatTableOp_succeed() { - Create_table_stmtContext context = getCreateTableContext( - "create table any_schema.abcd (id varchar(64)) format=(ENCODING='aaaa',LINE_DELIMITER=123,SKIP_HEADER=12," - + "EMPTY_FIELD_AS_NULL=true,NULL_IF_EXETERNAL=(1,2,3))"); + Create_table_stmtContext context = getCreateTableContext("create table any_schema.abcd (id varchar(64)) " + + "properties=(ACCESSID='abcd', PROJECT_NAME='ooop') " + + "partition_type=USER_SPECIFIED " + + "micro_index_clustered=true " + + "auto_refresh=OFF " + + "format=(ENCODING='aaaa',LINE_DELIMITER=123,SKIP_HEADER=12," + + "EMPTY_FIELD_AS_NULL=true,NULL_IF_EXETERNAL=(1,2,3))"); StatementFactory factory = new OracleCreateTableFactory(context); CreateTable actual = factory.generate(); @@ -380,6 +384,13 @@ public void generate_formatTableOp_succeed() { map.put("NULL_IF_EXETERNAL", es); map.put("LINE_DELIMITER", new ConstExpression("123")); tableOptions.setFormat(map); + tableOptions.setPartitionType("USER_SPECIFIED"); + tableOptions.setMicroIndexClustered(true); + tableOptions.setAutoRefresh("OFF"); + Map externalProperties = new HashMap<>(); + externalProperties.put("ACCESSID", "'abcd'"); + externalProperties.put("PROJECT_NAME", "'ooop'"); + tableOptions.setExternalProperties(externalProperties); expect.setTableOptions(tableOptions); Assert.assertEquals(expect, actual); } diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleExpressionFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleExpressionFactoryTest.java index 6276205688..36457234c1 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleExpressionFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleExpressionFactoryTest.java @@ -33,6 +33,7 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Bit_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.ExprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.PredicateContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Single_row_functionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Xml_functionContext; import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.Expression.ReferenceOperator; @@ -56,14 +57,14 @@ import com.oceanbase.tools.sqlparser.statement.expression.FullTextSearch; import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; import com.oceanbase.tools.sqlparser.statement.expression.FunctionParam; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.ScalarsMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.StrictMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.UniqueMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.WrapperMode; import com.oceanbase.tools.sqlparser.statement.expression.JsonKeyValue; import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption; import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption.OnMismatch; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.ScalarsMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.StrictMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.UniqueMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.WrapperMode; import com.oceanbase.tools.sqlparser.statement.expression.NullExpression; import com.oceanbase.tools.sqlparser.statement.expression.ParamWithAssign; import com.oceanbase.tools.sqlparser.statement.expression.RelationReference; @@ -1609,6 +1610,41 @@ public void generate_xmlParse_generateSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_spatialCellid_generateSucceed() { + Single_row_functionContext context = getSingleRowFunctionContext("spatial_cellid('aaa')"); + OracleExpressionFactory factory = new OracleExpressionFactory(); + Expression actual = factory.visit(context); + + FunctionCall expect = new FunctionCall("spatial_cellid", + Collections.singletonList(new ExpressionParam(new ConstExpression("'aaa'")))); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_spatialMbr_generateSucceed() { + Single_row_functionContext context = getSingleRowFunctionContext("spatial_mbr('aaa')"); + OracleExpressionFactory factory = new OracleExpressionFactory(); + Expression actual = factory.visit(context); + + FunctionCall expect = new FunctionCall("spatial_mbr", + Collections.singletonList(new ExpressionParam(new ConstExpression("'aaa'")))); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_sdoRelate_generateSucceed() { + Single_row_functionContext context = getSingleRowFunctionContext("sdo_relate('aaa', 123, 456)"); + OracleExpressionFactory factory = new OracleExpressionFactory(); + Expression actual = factory.visit(context); + + FunctionCall expect = new FunctionCall("sdo_relate", Arrays.asList( + new ExpressionParam(new ConstExpression("'aaa'")), + new ExpressionParam(new ConstExpression("123")), + new ExpressionParam(new ConstExpression("456")))); + Assert.assertEquals(expect, actual); + } + @Test public void generate_deleteXml_generateSucceed() { Xml_functionContext context = getXmlExprContext("deletexml(1,2,3)"); @@ -2032,7 +2068,7 @@ public void generate_jsonConstrain_generateSucceed() { Expression actual = factory.generate(); Expression left = new ConstExpression("'aaa'"); - JsonConstraint right = new JsonConstraint(); + JsonOption right = new JsonOption(); right.setStrictMode(StrictMode.LAX); right.setScalarsMode(ScalarsMode.ALLOW_SCALARS); right.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); @@ -2047,7 +2083,7 @@ public void generate_jsonConstrain1_generateSucceed() { Expression actual = factory.generate(); Expression left = new ConstExpression("'aaa'"); - JsonConstraint right = new JsonConstraint(); + JsonOption right = new JsonOption(); Expression expect = new CompoundExpression(left, right, Operator.EQ); Assert.assertEquals(expect, actual); } @@ -2107,7 +2143,9 @@ public void generate_jsonExists2_generateSucceed() { JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new BoolValue(true)); jsonOnOption.setOnEmpty(new ConstExpression("error_p")); - expect.addOption(jsonOnOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2122,7 +2160,9 @@ public void generate_accessFunc1_generateSucceed() { FunctionCall expect = new FunctionCall("func", Arrays.asList(p1, p2)); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new BoolValue(true)); - expect.addOption(jsonOnOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2135,9 +2175,11 @@ public void generate_accessFunc3_generateSucceed() { FunctionParam p1 = new ExpressionParam(new ConstExpression("12")); FunctionParam p2 = new ExpressionParam(new ConstExpression("12")); FunctionCall expect = new FunctionCall("func", Arrays.asList(p1, p2)); + JsonOption jsonOpt = new JsonOption(); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new ConstExpression("error_p")); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2167,10 +2209,10 @@ public void generate_jsonArrayAgg1_generateSucceed() { expect.addOption(new OrderBy(Collections.singletonList(s))); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new ConstExpression("absent")); - expect.addOption(jsonOnOption); expect.addOption(new GeneralDataType("raw", Collections.singletonList("12"))); - JsonConstraint c = new JsonConstraint(); + JsonOption c = new JsonOption(); c.setStrictMode(StrictMode.STRICT); + c.setOnOption(jsonOnOption); expect.addOption(c); Assert.assertEquals(expect, actual); } @@ -2210,11 +2252,11 @@ public void generate_jsonObjAgg1_generateSucceed() { expect.addOption(new ConstExpression("format json")); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new NullExpression()); - expect.addOption(jsonOnOption); expect.addOption(new CharacterType("nvarchar2", new BigDecimal("12"))); - JsonConstraint c = new JsonConstraint(); + JsonOption c = new JsonOption(); c.setStrictMode(StrictMode.STRICT); c.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); + c.setOnOption(jsonOnOption); expect.addOption(c); Assert.assertEquals(expect, actual); } @@ -2235,7 +2277,9 @@ public void generate_jsonExists3_generateSucceed() { JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new BoolValue(true)); jsonOnOption.setOnEmpty(new ConstExpression("error_p")); - expect.addOption(jsonOnOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2246,7 +2290,7 @@ public void generate_jsonConstrain2_generateSucceed() { Expression actual = factory.generate(); Expression left = new ConstExpression("'aaa'"); - JsonConstraint right = new JsonConstraint(); + JsonOption right = new JsonOption(); right.setStrictMode(StrictMode.STRICT); right.setScalarsMode(ScalarsMode.DISALLOW_SCALARS); right.setUniqueMode(UniqueMode.WITHOUT_UNIQUE_KEYS); @@ -2273,7 +2317,9 @@ public void generate_jsonObject1_generateSucceed() { FunctionCall expect = new FunctionCall("json_object", Collections.emptyList()); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new ConstExpression("absent")); - expect.addOption(jsonOnOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2297,11 +2343,11 @@ public void generate_jsonObject3_generateSucceed() { FunctionCall expect = new FunctionCall("json_object", Collections.emptyList()); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new NullExpression()); - expect.addOption(jsonOnOption); expect.addOption(new GeneralDataType("json", null)); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); jc.setStrictMode(StrictMode.STRICT); jc.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); + jc.setOnOption(jsonOnOption); expect.addOption(jc); Assert.assertEquals(expect, actual); } @@ -2316,11 +2362,11 @@ public void generate_jsonObject4_generateSucceed() { FunctionCall expect = new FunctionCall("json_object", Collections.singletonList(p)); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new NullExpression()); - expect.addOption(jsonOnOption); expect.addOption(new GeneralDataType("json", null)); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); jc.setStrictMode(StrictMode.STRICT); jc.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); + jc.setOnOption(jsonOnOption); expect.addOption(jc); Assert.assertEquals(expect, actual); } @@ -2344,7 +2390,7 @@ public void generate_jsonObject5_generateSucceed() { new ExpressionParam( new JsonKeyValue(new ConstExpression("' asdasdas '"), new RelationReference("col3", null))); FunctionCall expect = new FunctionCall("json_object", Arrays.asList(p, p1, p2, p3, p4)); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); jc.setStrictMode(StrictMode.STRICT); expect.addOption(jc); Assert.assertEquals(expect, actual); @@ -2357,7 +2403,7 @@ public void generate_jsonObject6_generateSucceed() { Expression actual = factory.generate(); FunctionCall expect = new FunctionCall("json_object", Collections.emptyList()); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); jc.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); expect.addOption(jc); Assert.assertEquals(expect, actual); @@ -2373,6 +2419,7 @@ public void generate_jsonQuery_generateSucceed() { p2.addOption(new ConstExpression("format json")); FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_query", Arrays.asList(p2, p3)); + expect.addOption(new JsonOption()); Assert.assertEquals(expect, actual); } @@ -2389,18 +2436,19 @@ public void generate_jsonQuery1_generateSucceed() { FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_query", Arrays.asList(p2, p3)); expect.addOption(new GeneralDataType("json", null)); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("pretty")); - expect.addOption(new ConstExpression("ascii")); - JsonConstraint jsonConstraint = new JsonConstraint(); - jsonConstraint.setScalarsMode(ScalarsMode.ALLOW_SCALARS); - jsonConstraint.setWrapperMode(WrapperMode.WITH_CONDITIONAL_WRAPPER); - expect.addOption(jsonConstraint); + + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setPretty(true); + jsonOpt.setAscii(true); + jsonOpt.setScalarsMode(ScalarsMode.ALLOW_SCALARS); + jsonOpt.setWrapperMode(WrapperMode.WITH_CONDITIONAL_WRAPPER); + expect.addOption(jsonOpt); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnMismatches(Collections.singletonList(new OnMismatch(new ConstExpression("dot"), null))); jsonOnOption.setOnEmpty(new ConstExpression("empty")); jsonOnOption.setOnError(new NullExpression()); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); Assert.assertEquals(expect, actual); } @@ -2426,12 +2474,12 @@ public void generate_jsonQuery2_generateSucceed() { p2.addOption(new ConstExpression("format json")); FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_query", Arrays.asList(p2, p3)); - JsonConstraint jsonConstraint = new JsonConstraint(); - jsonConstraint.setWrapperMode(WrapperMode.valueOf(s.replace(" ", "_"))); - expect.addOption(jsonConstraint); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(WrapperMode.valueOf(s.replace(" ", "_"))); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnMismatches(Collections.singletonList(new OnMismatch(new NullExpression(), null))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } } @@ -2445,8 +2493,10 @@ public void generate_jsonMergepatch_generateSucceed() { FunctionParam p2 = new ExpressionParam(new ConstExpression("'a'")); FunctionParam p3 = new ExpressionParam(new ConstExpression("2")); FunctionCall expect = new FunctionCall("json_mergepatch", Arrays.asList(p2, p3)); - expect.addOption(new ConstExpression("PRETTY")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setAscii(true); + jsonOpt.setPretty(true); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2460,11 +2510,13 @@ public void generate_jsonMergepatch1_generateSucceed() { FunctionParam p3 = new ExpressionParam(new ConstExpression("2")); FunctionCall expect = new FunctionCall("json_mergepatch", Arrays.asList(p2, p3)); expect.addOption(new GeneralDataType("json", null)); - expect.addOption(new ConstExpression("PRETTY")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setPretty(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new NullExpression()); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2482,12 +2534,14 @@ public void generate_jsonMergepatch2_generateSucceed() { CharacterType characterType = new CharacterType("varchar2", new BigDecimal("12")); characterType.setBinary(true); expect.addOption(characterType); - expect.addOption(new ConstExpression("TRUNCATE")); - expect.addOption(new ConstExpression("PRETTY")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); + jsonOpt.setPretty(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new ConstExpression("error_p")); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2526,8 +2580,10 @@ public void generate_jsonArray2_generateSucceed() { FunctionCall expect = new FunctionCall("json_array", Arrays.asList(p2, p3)); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new ConstExpression("absent")); - expect.addOption(jsonOnOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(jsonOnOption); expect.addOption(new GeneralDataType("json", null)); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2543,9 +2599,9 @@ public void generate_jsonArray3_generateSucceed() { FunctionCall expect = new FunctionCall("json_array", Arrays.asList(p2, p3)); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new NullExpression()); - expect.addOption(jsonOnOption); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); jc.setStrictMode(StrictMode.STRICT); + jc.setOnOption(jsonOnOption); expect.addOption(jc); Assert.assertEquals(expect, actual); } @@ -2560,6 +2616,7 @@ public void generate_jsonValue_generateSucceed() { p2.addOption(new ConstExpression("format json")); FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_value", Arrays.asList(p2, p3)); + expect.addOption(new JsonOption()); Assert.assertEquals(expect, actual); } @@ -2575,15 +2632,17 @@ public void generate_jsonValue1_generateSucceed() { FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_value", Arrays.asList(p2, p3)); expect.addOption(new CharacterType("nchar", new BigDecimal("2"))); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("ascii")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new ConstExpression("1")); jsonOnOption.setOnEmpty(new NullExpression()); jsonOnOption.setOnMismatches(Arrays.asList(new OnMismatch(new ConstExpression("ignore"), Collections.singletonList("MISSING DATA")), new OnMismatch(new NullExpression(), Collections.emptyList()))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2601,15 +2660,18 @@ public void generate_jsonValue2_generateSucceed() { CharacterType characterType = new CharacterType("varchar2", new BigDecimal("2")); characterType.setBinary(true); expect.addOption(characterType); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("ascii")); + + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnEmpty(new ConstExpression("1")); jsonOnOption.setOnError(new ConstExpression("error_p")); jsonOnOption.setOnMismatches(Arrays.asList(new OnMismatch(new ConstExpression("ignore"), Collections.singletonList("MISSING DATA")), new OnMismatch(new NullExpression(), Collections.emptyList()))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2626,15 +2688,17 @@ public void generate_jsonValue4_generateSucceed() { FunctionCall expect = new FunctionCall("json_value", Arrays.asList(p2, p3)); CharacterType characterType = new CharacterType("char", null); expect.addOption(characterType); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("ascii")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnEmpty(new ConstExpression("1")); jsonOnOption.setOnError(new ConstExpression("error_p")); jsonOnOption.setOnMismatches(Arrays.asList(new OnMismatch(new ConstExpression("ignore"), Collections.singletonList("MISSING DATA")), new OnMismatch(new NullExpression(), Collections.emptyList()))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2650,15 +2714,17 @@ public void generate_jsonValue5_generateSucceed() { FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_value", Arrays.asList(p2, p3)); expect.addOption(new GeneralDataType("raw", null)); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("ascii")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnEmpty(new ConstExpression("1")); jsonOnOption.setOnError(new ConstExpression("error_p")); jsonOnOption.setOnMismatches(Arrays.asList(new OnMismatch(new ConstExpression("ignore"), Collections.singletonList("MISSING DATA")), new OnMismatch(new NullExpression(), Collections.emptyList()))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2675,15 +2741,17 @@ public void generate_jsonValue3_generateSucceed() { FunctionCall expect = new FunctionCall("json_value", Arrays.asList(p2, p3)); CharacterType characterType = new CharacterType("nvarchar2", null); expect.addOption(characterType); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("ascii")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnEmpty(new ConstExpression("1")); jsonOnOption.setOnError(new ConstExpression("error_p")); jsonOnOption.setOnMismatches(Arrays.asList(new OnMismatch(new ConstExpression("ignore"), Collections.singletonList("MISSING DATA")), new OnMismatch(new NullExpression(), Collections.emptyList()))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2756,50 +2824,59 @@ public void generate_jsonTable2_generateSucceed() { p1.addOption(new ConstExpression("format json")); FunctionParam p2 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_table", Arrays.asList(p1, p2)); + JsonOption jsonOpt = new JsonOption(); JsonOnOption onOption = new JsonOnOption(); onOption.setOnError(new ConstExpression("error_p")); onOption.setOnEmpty(new NullExpression()); - expect.addOption(onOption); + jsonOpt.setOnOption(onOption); + expect.addOption(jsonOpt); FunctionParam op1 = new ExpressionParam(new ColumnReference(null, null, "\"abcd\"")); op1.addOption(new ConstExpression("FOR ORDINALITY")); expect.addOption(op1); FunctionParam op2 = new ExpressionParam(new ColumnReference(null, null, "col1")); op2.addOption(new NumberType("int", null, null)); - op2.addOption(new ConstExpression("truncate")); + JsonOption jsonOpt1 = new JsonOption(); + jsonOpt1.setAsis(true); + jsonOpt1.setTruncate(true); op2.addOption(new ConstExpression("exists")); op2.addOption(new ConstExpression("123")); onOption = new JsonOnOption(); onOption.setOnEmpty(new BoolValue(true)); - op2.addOption(onOption); + jsonOpt1.setOnOption(onOption); + op2.addOption(jsonOpt1); expect.addOption(op2); FunctionParam op3 = new ExpressionParam(new ColumnReference(null, null, "col2")); op3.addOption(new GeneralDataType("json", null)); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); + jc.setAsis(true); jc.setScalarsMode(ScalarsMode.DISALLOW_SCALARS); jc.setWrapperMode(WrapperMode.WITH_CONDITIONAL_ARRAY_WRAPPER); - op3.addOption(jc); op3.addOption(new ColumnReference(null, null, "col21")); onOption = new JsonOnOption(); onOption.setOnEmpty(new ConstExpression("empty")); - op3.addOption(onOption); + jc.setOnOption(onOption); + op3.addOption(jc); expect.addOption(op3); FunctionParam op4 = new ExpressionParam(new ColumnReference(null, null, "col3")); op4.addOption(new GeneralDataType("blob", null)); op4.addOption(new ConstExpression("format json")); - op4.addOption(new ConstExpression("truncate")); - jc = new JsonConstraint(); + jc = new JsonOption(); + jc.setAsis(true); + jc.setTruncate(true); jc.setScalarsMode(ScalarsMode.ALLOW_SCALARS); jc.setWrapperMode(WrapperMode.WITH_ARRAY_WRAPPER); - op4.addOption(jc); op4.addOption(new ColumnReference(null, null, "col31")); - op4.addOption(onOption); + op4.addOption(jc); + jc.setOnOption(onOption); expect.addOption(op4); FunctionParam op5 = new ExpressionParam(new ColumnReference(null, null, "col4")); op5.addOption(new CharacterType("nchar", new BigDecimal("12"))); - op5.addOption(new ConstExpression("truncate")); + jc = new JsonOption(); + jc.setTruncate(true); + jc.setAsis(true); ColumnReference rc = new ColumnReference(null, null, "col41"); CollectionExpression es = new CollectionExpression(); es.addExpression(new ConstExpression("*")); @@ -2807,7 +2884,8 @@ public void generate_jsonTable2_generateSucceed() { op5.addOption(rc); onOption = new JsonOnOption(); onOption.setOnEmpty(new CompoundExpression(new ConstExpression("3"), null, Operator.SUB)); - op5.addOption(onOption); + jc.setOnOption(onOption); + op5.addOption(jc); expect.addOption(op5); FunctionParam op6 = new ExpressionParam(new ConstExpression("nested path")); op6.addOption(new ConstExpression("123")); @@ -2833,48 +2911,57 @@ public void generate_jsonTable3_generateSucceed() { p1.addOption(new ConstExpression("format json")); FunctionParam p2 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_table", Arrays.asList(p1, p2)); + JsonOption jsonOpt = new JsonOption(); JsonOnOption onOption = new JsonOnOption(); onOption.setOnError(new ConstExpression("error_p")); onOption.setOnEmpty(new CompoundExpression(new ConstExpression("5"), null, Operator.SUB)); - expect.addOption(onOption); + jsonOpt.setOnOption(onOption); + expect.addOption(jsonOpt); FunctionParam op1 = new ExpressionParam(new ColumnReference(null, null, "\"abcd\"")); op1.addOption(new ConstExpression("FOR ORDINALITY")); expect.addOption(op1); FunctionParam op2 = new ExpressionParam(new ColumnReference(null, null, "col1")); op2.addOption(new NumberType("int", null, null)); - op2.addOption(new ConstExpression("truncate")); + JsonOption jsonOpt1 = new JsonOption(); + jsonOpt1.setAsis(true); + jsonOpt1.setTruncate(true); op2.addOption(new ConstExpression("exists")); op2.addOption(new ConstExpression("123")); onOption = new JsonOnOption(); onOption.setOnEmpty(new BoolValue(true)); - op2.addOption(onOption); + jsonOpt1.setOnOption(onOption); + op2.addOption(jsonOpt1); expect.addOption(op2); FunctionParam op3 = new ExpressionParam(new ColumnReference(null, null, "col2")); op3.addOption(new GeneralDataType("json", null)); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); + jc.setAsis(true); jc.setWrapperMode(WrapperMode.WITH_CONDITIONAL_ARRAY_WRAPPER); - op3.addOption(jc); op3.addOption(new ColumnReference(null, null, "col21")); onOption = new JsonOnOption(); onOption.setOnEmpty(new ConstExpression("empty")); - op3.addOption(onOption); + jc.setOnOption(onOption); + op3.addOption(jc); expect.addOption(op3); FunctionParam op4 = new ExpressionParam(new ColumnReference(null, null, "col3")); op4.addOption(new GeneralDataType("blob", null)); op4.addOption(new ConstExpression("format json")); - op4.addOption(new ConstExpression("truncate")); - jc = new JsonConstraint(); + jc = new JsonOption(); + jc.setAsis(true); + jc.setTruncate(true); jc.setScalarsMode(ScalarsMode.ALLOW_SCALARS); - op4.addOption(jc); op4.addOption(new ColumnReference(null, null, "col31")); - op4.addOption(onOption); + op4.addOption(jc); + jc.setOnOption(onOption); expect.addOption(op4); FunctionParam op5 = new ExpressionParam(new ColumnReference(null, null, "col4")); op5.addOption(new CharacterType("nchar", new BigDecimal("12"))); - op5.addOption(new ConstExpression("truncate")); + jc = new JsonOption(); + jc.setTruncate(true); + jc.setAsis(true); ColumnReference rc = new ColumnReference(null, null, "col41"); CollectionExpression es = new CollectionExpression(); es.addExpression(new ConstExpression("*")); @@ -2882,7 +2969,8 @@ public void generate_jsonTable3_generateSucceed() { op5.addOption(rc); onOption = new JsonOnOption(); onOption.setOnEmpty(new CompoundExpression(new ConstExpression("3"), null, Operator.SUB)); - op5.addOption(onOption); + jc.setOnOption(onOption); + op5.addOption(jc); expect.addOption(op5); FunctionParam op6 = new ExpressionParam(new ConstExpression("nested path")); op6.addOption(new ConstExpression("123")); @@ -2900,9 +2988,11 @@ public void generate_jsonEqualExpr_generateSucceed() { ExpressionParam p1 = new ExpressionParam(new ConstExpression("'[1,]'")); ExpressionParam p2 = new ExpressionParam(new ConstExpression("'[1]'")); FunctionCall expect = new FunctionCall("json_equal", Arrays.asList(p1, p2)); + JsonOption jsonOpt = new JsonOption(); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new BoolValue(false)); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2930,6 +3020,14 @@ private ExprContext getExprContext(String expr) { return parser.expr(); } + private Single_row_functionContext getSingleRowFunctionContext(String expr) { + OBLexer lexer = new OBLexer(CharStreams.fromString(expr)); + CommonTokenStream tokens = new CommonTokenStream(lexer); + OBParser parser = new OBParser(tokens); + parser.setErrorHandler(new BailErrorStrategy()); + return parser.single_row_function(); + } + private Xml_functionContext getXmlExprContext(String expr) { OBLexer lexer = new OBLexer(CharStreams.fromString(expr)); CommonTokenStream tokens = new CommonTokenStream(lexer); diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleInsertFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleInsertFactoryTest.java index 5657ef167b..d7ecc02e1b 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleInsertFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleInsertFactoryTest.java @@ -16,10 +16,7 @@ package com.oceanbase.tools.sqlparser.adapter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -83,6 +80,44 @@ public void generate_simpleInsert_succeed() { Assert.assertEquals(actual, expect); } + @Test + public void generate_simpleInsertWithPartition_succeed() { + StatementFactory factory = new OracleInsertFactory( + getInsertContext("insert into a.b partition(col='111', col2=121) alias values(1,default)")); + Insert actual = factory.generate(); + + RelationFactor factor = new RelationFactor("b"); + factor.setSchema("a"); + InsertTable insertTable = new InsertTable(factor); + Map parti = new HashMap<>(); + parti.put("col", new ConstExpression("'111'")); + parti.put("col2", new ConstExpression("121")); + insertTable.setPartitionUsage(new PartitionUsage(PartitionType.PARTITION, parti)); + insertTable.setAlias("alias"); + List> values = new ArrayList<>(); + values.add(Arrays.asList(new ConstExpression("1"), new ConstExpression("default"))); + insertTable.setValues(values); + Insert expect = new Insert(Collections.singletonList(insertTable), null); + Assert.assertEquals(actual, expect); + } + + @Test + public void generate_overwriteInsert_succeed() { + StatementFactory factory = new OracleInsertFactory( + getInsertContext("insert overwrite a.b values(1,default)")); + Insert actual = factory.generate(); + + RelationFactor factor = new RelationFactor("b"); + factor.setSchema("a"); + InsertTable insertTable = new InsertTable(factor); + List> values = new ArrayList<>(); + values.add(Arrays.asList(new ConstExpression("1"), new ConstExpression("default"))); + insertTable.setValues(values); + Insert expect = new Insert(Collections.singletonList(insertTable), null); + expect.setOverwrite(true); + Assert.assertEquals(expect, actual); + } + @Test public void generate_insertSelect_succeed() { StatementFactory factory = new OracleInsertFactory( From 0ba1d21b43b43a419bca4c1a04acea0bb57e6ee0 Mon Sep 17 00:00:00 2001 From: yizhou Date: Fri, 1 Nov 2024 19:52:02 +0800 Subject: [PATCH 018/118] secure(framework): enable secure cookie for http session (#3781) * secure(framework): offer cookie secure option in system configurations * fix secure cookie not works --- .../src/main/resources/config/application.yml | 4 ++++ server/odc-server/src/main/resources/data.sql | 8 +++++++- .../com/oceanbase/odc/config/BeanConfiguration.java | 11 ----------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/server/odc-server/src/main/resources/config/application.yml b/server/odc-server/src/main/resources/config/application.yml index 7321a8e19c..fc2ebd0b43 100644 --- a/server/odc-server/src/main/resources/config/application.yml +++ b/server/odc-server/src/main/resources/config/application.yml @@ -53,6 +53,10 @@ server: servlet: session: timeout: 8h + cookie: + name: JSESSIONID + http-only: true + secure: true encoding: charset: UTF-8 tomcat: diff --git a/server/odc-server/src/main/resources/data.sql b/server/odc-server/src/main/resources/data.sql index cf0904f719..84e0223c4a 100644 --- a/server/odc-server/src/main/resources/data.sql +++ b/server/odc-server/src/main/resources/data.sql @@ -844,4 +844,10 @@ ON DUPLICATE KEY UPDATE `id`=`id`; INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('odc.integration.git.repository-retention-minutes', '1440', 'The longest time ODC can cache repository copies') ON DUPLICATE KEY UPDATE `id`=`id`; INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('odc.integration.git.repository-max-cached-size', - '1000', 'The maximum number of repository copies cached in ODC') ON DUPLICATE KEY UPDATE `id`=`id`; \ No newline at end of file + '1000', 'The maximum number of repository copies cached in ODC') ON DUPLICATE KEY UPDATE `id`=`id`; + + -- + -- v4.3.3 + -- + INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('server.servlet.session.cookie.secure', + 'true', 'Enable secure cookie or not, default value true') ON DUPLICATE KEY UPDATE `id`=`id`; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/config/BeanConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/config/BeanConfiguration.java index a4b60b267a..8ea01f8a72 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/config/BeanConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/config/BeanConfiguration.java @@ -28,13 +28,10 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.web.firewall.DefaultHttpFirewall; import org.springframework.security.web.firewall.HttpFirewall; -import org.springframework.session.web.http.CookieSerializer; -import org.springframework.session.web.http.DefaultCookieSerializer; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.BeanProperty; @@ -111,14 +108,6 @@ public CloudMetadataClient cloudMetadataClient() { return new NullCloudMetadataClient(); } - @Bean - @Primary - public CookieSerializer cookieSerializer() { - DefaultCookieSerializer serializer = new DefaultCookieSerializer(); - serializer.setCookieName("JSESSIONID"); - return serializer; - } - @Slf4j public static class NullCloudMetadataClient implements CloudMetadataClient { From cf23bf319bca71435a675e7aada8cfaeaa8df163 Mon Sep 17 00:00:00 2001 From: "longpeng.zlp" Date: Thu, 7 Nov 2024 14:44:54 +0800 Subject: [PATCH 019/118] mrege 4.3.2 to 4.3.x --- .../task/service/StdTaskFrameworkService.java | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java index 89331794dd..efcf69ec07 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java @@ -350,6 +350,43 @@ public void handleResult(TaskResult taskResult) { handleTaskResultInner(je, taskResult); } + private void doRefreshResult(Long id) throws JobException { + JobEntity je = find(id); + // CANCELING is also a state within the running phase + if (JobStatus.RUNNING != je.getStatus()) { + log.info("Job is not running, ignore refresh, jobId={}, currentStatus={}", id, je.getStatus()); + return; + } + + String executorEndpoint = executorEndpointManager.getExecutorEndpoint(je); + DefaultTaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); + if (result.getStatus() == TaskStatus.PREPARING) { + log.info("Job is preparing, ignore refresh, jobId={}, currentStatus={}", id, result.getStatus()); + return; + } + DefaultTaskResult previous = JsonUtils.fromJson(je.getResultJson(), DefaultTaskResult.class); + + if (!updateHeartbeatTime(id)) { + log.warn("Update lastHeartbeatTime failed, the job may finished or deleted already, jobId={}", id); + return; + } + if (!result.progressChanged(previous)) { + log.info("Progress not changed, skip update result to metadb, jobId={}, currentProgress={}", + id, result.getProgress()); + return; + } + log.info("Progress changed, will update result, jobId={}, currentProgress={}", id, result.getProgress()); + handleTaskResult(je.getJobType(), result); + saveOrUpdateLogMetadata(result, je.getId(), je.getStatus()); + + if (result.getStatus().isTerminated() && MapUtils.isEmpty(result.getLogMetadata())) { + log.info("Job is finished but log have not uploaded, continue monitor result, jobId={}, currentStatus={}", + je.getId(), je.getStatus()); + return; + } + handleTaskResultInner(je, result); + } + protected void handleTaskResultInner(JobEntity jobEntity, TaskResult result) { JobStatus expectedJobStatus = jobStatusFsm.determinateJobStatus(jobEntity.getStatus(), result.getStatus()); int rows = updateTaskResult(result, jobEntity, expectedJobStatus); @@ -435,43 +472,6 @@ protected void tryReleaseResource(JobEntity jobEntity, boolean isJobDone) { } } - private void doRefreshResult(Long id) throws JobException { - JobEntity je = find(id); - // CANCELING is also a state within the running phase - if (JobStatus.RUNNING != je.getStatus()) { - log.info("Job is not running, ignore refresh, jobId={}, currentStatus={}", id, je.getStatus()); - return; - } - - String executorEndpoint = executorEndpointManager.getExecutorEndpoint(je); - DefaultTaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); - if (result.getStatus() == TaskStatus.PREPARING) { - log.info("Job is preparing, ignore refresh, jobId={}, currentStatus={}", id, result.getStatus()); - return; - } - DefaultTaskResult previous = JsonUtils.fromJson(je.getResultJson(), DefaultTaskResult.class); - - if (!updateHeartbeatTime(id)) { - log.warn("Update lastHeartbeatTime failed, the job may finished or deleted already, jobId={}", id); - return; - } - if (!result.progressChanged(previous)) { - log.info("Progress not changed, skip update result to metadb, jobId={}, currentProgress={}", - id, result.getProgress()); - return; - } - log.info("Progress changed, will update result, jobId={}, currentProgress={}", id, result.getProgress()); - handleTaskResult(je.getJobType(), result); - saveOrUpdateLogMetadata(result, je.getId(), je.getStatus()); - - if (result.getStatus().isTerminated() && MapUtils.isEmpty(result.getLogMetadata())) { - log.info("Job is finished but log have not uploaded, continue monitor result, jobId={}, currentStatus={}", - je.getId(), je.getStatus()); - return; - } - handleTaskResultInner(je, result); - } - private boolean updateHeartbeatTime(Long id) { int rows = jobRepository.updateHeartbeatTime(id, JobStatus.RUNNING); if (rows > 0) { From 45be1de962bfef056a5d44974d11d476ddaabba0 Mon Sep 17 00:00:00 2001 From: "longpeng.zlp" Date: Thu, 7 Nov 2024 14:57:24 +0800 Subject: [PATCH 020/118] mrege 4.3.2 to 4.3.x --- .../odc/service/task/executor/task/BaseTaskTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java index 4ded74cdef..a60dfeb544 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java @@ -26,10 +26,13 @@ import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.service.task.ExceptionListener; import com.oceanbase.odc.service.task.TaskContext; +import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.caller.DefaultJobContext; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; -import com.oceanbase.odc.service.task.executor.server.TaskMonitor; +import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.DefaultTaskResultBuilder; +import com.oceanbase.odc.service.task.executor.TaskMonitor; import com.oceanbase.odc.service.task.schedule.JobIdentity; /** From 860d0303f557c5980571271ce457a1719f2b60f4 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Fri, 8 Nov 2024 10:33:56 +0800 Subject: [PATCH 021/118] fix(external table):get table and external table together without exception (#3793) * can get table and external table together without exception * Refactor code * Add exception handling for external table support check * Refactor method isExternalTableSupported --- .../connection/table/TableService.java | 20 ++++++++++++---- .../db/schema/DBSchemaSyncService.java | 2 +- .../db/schema/syncer/DBSchemaSyncer.java | 5 ++-- .../syncer/column/AbstractDBColumnSyncer.java | 2 +- .../syncer/object/AbstractDBObjectSyncer.java | 2 +- .../syncer/object/DBExternalTableSyncer.java | 23 +++++++++++++++++++ .../feature/VersionDiffConfigService.java | 16 +++++++++++++ 7 files changed, 61 insertions(+), 9 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java index 3aba1649d8..0e3eef06b8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java @@ -49,6 +49,7 @@ import com.oceanbase.odc.core.shared.exception.UnsupportedException; import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint; import com.oceanbase.odc.plugin.schema.api.TableExtensionPoint; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; @@ -59,9 +60,11 @@ import com.oceanbase.odc.service.db.schema.syncer.DBSchemaSyncer; import com.oceanbase.odc.service.db.schema.syncer.object.DBExternalTableSyncer; import com.oceanbase.odc.service.db.schema.syncer.object.DBTableSyncer; +import com.oceanbase.odc.service.feature.VersionDiffConfigService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; @@ -79,6 +82,9 @@ @Validated public class TableService { + @Autowired + private VersionDiffConfigService versionDiffConfigService; + @Autowired private DatabaseService databaseService; @@ -126,11 +132,17 @@ public List
list(@NonNull @Valid QueryTableParams params) throws SQLExcep tableExtension); } if (types.contains(DBObjectType.EXTERNAL_TABLE)) { - generateListAndSyncDBTablesByTableType(params, database, dataSource, tables, conn, - DBObjectType.EXTERNAL_TABLE, tableExtension); + InformationExtensionPoint point = + ConnectionPluginUtil.getInformationExtension(dataSource.getDialectType()); + String databaseProductVersion = point.getDBVersion(conn); + if (versionDiffConfigService.isExternalTableSupported(dataSource.getDialectType(), + databaseProductVersion)) { + generateListAndSyncDBTablesByTableType(params, database, dataSource, tables, conn, + DBObjectType.EXTERNAL_TABLE, tableExtension); + } } - return tables; } + return tables; } private void generateListAndSyncDBTablesByTableType(QueryTableParams params, Database database, @@ -189,7 +201,7 @@ private void syncDBTables(@NotNull Connection connection, @NotNull Database data new Object[] {ResourceType.ODC_TABLE.getLocalizedMessage()}, "Can not acquire jdbc lock"); } try { - if (syncer.supports(dialectType)) { + if (syncer.supports(dialectType, connection)) { syncer.sync(connection, database, dialectType); } else { throw new UnsupportedException("Unsupported dialect type: " + dialectType); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java index 2d00606786..466b6ee216 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java @@ -90,7 +90,7 @@ public boolean sync(@NonNull Database database) throws InterruptedException, SQL Connection conn = dataSource.getConnection()) { boolean success = true; for (DBSchemaSyncer syncer : syncers) { - if (syncer.supports(config.getDialectType())) { + if (syncer.supports(config.getDialectType(), conn)) { try { syncer.sync(conn, database, config.getDialectType()); } catch (UnsupportedOperationException | UnsupportedException | NotImplementedException e) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/DBSchemaSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/DBSchemaSyncer.java index b9750d0899..3033e32d62 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/DBSchemaSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/DBSchemaSyncer.java @@ -42,11 +42,12 @@ public interface DBSchemaSyncer extends Ordered { /** * Check if the synchronizer supports the dialect type - * + * * @param dialectType dialect type, refer to {@link DialectType} + * @param connection * @return true if the synchronizer supports the dialect type, otherwise false */ - boolean supports(@NonNull DialectType dialectType); + boolean supports(@NonNull DialectType dialectType, @NonNull Connection connection); /** * Get the object type that the synchronizer supports diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java index 28962964a9..8e99f893b0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java @@ -115,7 +115,7 @@ public void sync(@NonNull Connection connection, @NonNull Database database, @No } @Override - public boolean supports(@NonNull DialectType dialectType) { + public boolean supports(@NonNull DialectType dialectType, @NonNull Connection connection) { return getExtensionPoint(dialectType) != null; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java index a84bba2702..294ec0a31d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java @@ -89,7 +89,7 @@ public void sync(@NonNull Connection connection, @NonNull Database database, @No } @Override - public boolean supports(@NonNull DialectType dialectType) { + public boolean supports(@NonNull DialectType dialectType, @NonNull Connection connection) { return getExtensionPoint(dialectType) != null; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalTableSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalTableSyncer.java index b687505d52..fc2ca837ae 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalTableSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalTableSyncer.java @@ -23,16 +23,21 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.metadb.iam.PermissionEntity; import com.oceanbase.odc.metadb.iam.PermissionRepository; import com.oceanbase.odc.metadb.iam.UserPermissionRepository; +import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint; import com.oceanbase.odc.plugin.schema.api.TableExtensionPoint; import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.feature.VersionDiffConfigService; +import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; import com.oceanbase.tools.dbbrowser.model.DBObjectType; import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; /** * @description: @@ -41,6 +46,7 @@ * @since: 4.3.3 */ @Component +@Slf4j public class DBExternalTableSyncer extends AbstractDBObjectSyncer { @Autowired @@ -49,6 +55,9 @@ public class DBExternalTableSyncer extends AbstractDBObjectSyncer toBeDeletedIds) { List permissions = @@ -58,6 +67,20 @@ protected void preDelete(@NonNull Set toBeDeletedIds) { userPermissionRepository.deleteByPermissionIds(permissionIds); } + @Override + public boolean supports(@NonNull DialectType dialectType, @NonNull Connection connection) { + try { + InformationExtensionPoint point = + ConnectionPluginUtil.getInformationExtension(dialectType); + String databaseProductVersion = point.getDBVersion(connection); + return versionDiffConfigService.isExternalTableSupported(dialectType, databaseProductVersion) + && getExtensionPoint(dialectType) != null; + } catch (Exception e) { + log.warn("check external table support failed", e); + return false; + } + } + @Override public DBObjectType getObjectType() { return DBObjectType.EXTERNAL_TABLE; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java index e9b5e3931d..c94bf8a35c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcOperations; import org.springframework.stereotype.Service; @@ -48,6 +49,7 @@ public class VersionDiffConfigService { private static final String SUPPORT_PROCEDURE = "support_procedure"; private static final String SUPPORT_FUNCTION = "support_function"; private static final String SUPPORT_PL_DEBUG = "support_pl_debug"; + private static final String SUPPORT_EXTERNAL_TABLE = "support_external_table"; private static final String COLUMN_DATA_TYPE = "column_data_type"; private static final String ARM_OB_PREFIX = "aarch64"; private static final String ARM_OB_SUPPORT_PL_DEBUG_MIN_VERSION = "3.2.3"; @@ -135,6 +137,20 @@ public List getSupportFeatures(ConnectionSession connectionSession) { return obSupportList; } + public boolean isExternalTableSupported(@NonNull DialectType dialectType, @NonNull String versionNumber) { + VersionDiffConfig config = new VersionDiffConfig(); + config.setDbMode(dialectType.name()); + config.setConfigKey(SUPPORT_EXTERNAL_TABLE); + List list = versionDiffConfigDAO.query(config); + String minVersion = CollectionUtils.isNotEmpty(list) ? list.get(0).getMinVersion() : null; + if ((dialectType == DialectType.OB_MYSQL || dialectType == DialectType.OB_ORACLE) + && minVersion != null + && VersionUtils.isGreaterThanOrEqualsTo(versionNumber, minVersion)) { + return true; + } + return false; + } + private boolean isHourFormat(ConnectionSession connectionSession) { JdbcOperations jdbcOperations = connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); From 435011c4cff649cf2ac825e6884d1534c1838394 Mon Sep 17 00:00:00 2001 From: LioRoger Date: Fri, 8 Nov 2024 10:52:43 +0800 Subject: [PATCH 022/118] feat(task): refactor base task, let it not be force depended any more (#3796) --- .../odc/service/task/TaskApplicationTest.java | 4 +- .../oceanbase/odc/core/alarm/AlarmUtils.java | 1 + server/odc-server/pom.xml | 10 ++ .../oceanbase/odc/agent/TaskApplication.java | 4 +- .../agent/runtime/ExecutorRequestHandler.java | 15 ++- .../odc/agent/runtime/TaskExecutor.java | 6 +- .../odc/agent/runtime}/TaskMonitor.java | 80 ++++++++++---- .../odc/agent/runtime/TaskRuntimeInfo.java | 33 ++++++ .../agent/runtime/ThreadPoolTaskExecutor.java | 92 +++++++++++++--- .../odc/agent/runtime/TaskMonitorTest.java | 102 ++++++++++++++++++ .../odc/service/task/TaskContext.java | 15 +++ .../odc/service/task/TaskEventListener.java | 55 ++++++++++ .../odc/service/task/base/BaseTask.java | 94 ++++++---------- .../databasechange/DatabaseChangeTask.java | 6 +- .../task/base/precheck/PreCheckTask.java | 2 +- .../task/base/rollback/RollbackPlanTask.java | 4 +- .../task/base/sqlplan/SqlPlanTask.java | 4 +- .../executor/DefaultTaskResultBuilder.java | 7 +- .../schedule/daemon/DestroyResourceJob.java | 2 +- .../task/executor/task/BaseTaskTest.java | 46 ++++++-- 20 files changed, 449 insertions(+), 133 deletions(-) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/executor => odc-server/src/main/java/com/oceanbase/odc/agent/runtime}/TaskMonitor.java (78%) create mode 100644 server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskRuntimeInfo.java create mode 100644 server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskEventListener.java diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java index 1b0ba39479..915d0c390a 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java @@ -25,6 +25,7 @@ import com.oceanbase.odc.TestConnectionUtil; import com.oceanbase.odc.agent.TaskApplication; +import com.oceanbase.odc.agent.runtime.TaskRuntimeInfo; import com.oceanbase.odc.agent.runtime.ThreadPoolTaskExecutor; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.PreConditions; @@ -79,7 +80,8 @@ private void assertRunningResult(JobIdentity ji) { try { Thread.sleep(60 * 1000L); - Task task = ThreadPoolTaskExecutor.getInstance().getTask(ji); + TaskRuntimeInfo taskRuntimeInfo = ThreadPoolTaskExecutor.getInstance().getTaskRuntimeInfo(ji); + Task task = taskRuntimeInfo.getTask(); Assert.assertSame(JobStatus.DONE, task.getStatus()); } catch (InterruptedException e) { throw new RuntimeException(e); diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmUtils.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmUtils.java index 3f42915589..236187fa8d 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmUtils.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmUtils.java @@ -39,6 +39,7 @@ public final class AlarmUtils { */ public static final String TASK_JOB_ID_NAME = "JobId"; public static final String RESOURCE_ID_NAME = "ResourceID"; + public static final String RESOURCE_TYPE = "ResourceType"; public static final String TASK_TYPE_NAME = "TaskType"; public static final String SCHEDULE_ID_NAME = "ScheduleId"; public static final Collection TASK_FRAMEWORK_ALARM_DIGEST_NAMES = diff --git a/server/odc-server/pom.xml b/server/odc-server/pom.xml index 078cf601c6..188ee7f13d 100644 --- a/server/odc-server/pom.xml +++ b/server/odc-server/pom.xml @@ -138,6 +138,16 @@ com.oceanbase odc-migrate + + junit + junit + test + + + org.mockito + mockito-core + test + diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/TaskApplication.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/TaskApplication.java index f483de2df8..13b83693dc 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/TaskApplication.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/TaskApplication.java @@ -33,7 +33,7 @@ import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.core.shared.Verify; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.caller.JobEnvironmentEncryptor; import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; @@ -63,7 +63,7 @@ public void run(String[] args) { try { server.start(); log.info("Starting embed server."); - BaseTask task = TaskFactory.create(context.getJobClass()); + Task task = TaskFactory.create(context.getJobClass()); ThreadPoolTaskExecutor.getInstance().execute(task, context); ExitHelper.await(); } catch (Exception e) { diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java index 02b5a61383..f7674798b0 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java @@ -25,12 +25,9 @@ import com.oceanbase.odc.service.common.response.Responses; import com.oceanbase.odc.service.common.response.SuccessResponse; import com.oceanbase.odc.service.common.util.UrlUtils; -import com.oceanbase.odc.service.task.Task; -import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.constants.JobExecutorUrls; import com.oceanbase.odc.service.task.executor.DefaultTaskResult; import com.oceanbase.odc.service.task.executor.DefaultTaskResultBuilder; -import com.oceanbase.odc.service.task.executor.TaskMonitor; import com.oceanbase.odc.service.task.executor.logger.LogBiz; import com.oceanbase.odc.service.task.executor.logger.LogBizImpl; import com.oceanbase.odc.service.task.executor.logger.LogUtils; @@ -89,21 +86,21 @@ public SuccessResponse process(HttpMethod httpMethod, String uri, String matcher = modifyParametersPattern.matcher(path); if (matcher.find()) { JobIdentity ji = getJobIdentity(matcher); - Task task = ThreadPoolTaskExecutor.getInstance().getTask(ji); - boolean result = task.modify(JobUtils.fromJsonToMap(requestData)); + TaskRuntimeInfo runtimeInfo = ThreadPoolTaskExecutor.getInstance().getTaskRuntimeInfo(ji); + boolean result = runtimeInfo.getTask().modify(JobUtils.fromJsonToMap(requestData)); return Responses.ok(result); } matcher = getResultPattern.matcher(path); if (matcher.find()) { JobIdentity ji = getJobIdentity(matcher); - BaseTask task = ThreadPoolTaskExecutor.getInstance().getTask(ji); - TaskMonitor taskMonitor = task.getTaskMonitor(); - DefaultTaskResult result = DefaultTaskResultBuilder.build(task); + TaskRuntimeInfo runtimeInfo = ThreadPoolTaskExecutor.getInstance().getTaskRuntimeInfo(ji); + TaskMonitor taskMonitor = runtimeInfo.getTaskMonitor(); + DefaultTaskResult result = DefaultTaskResultBuilder.build(runtimeInfo.getTask()); if (taskMonitor != null && MapUtils.isNotEmpty(taskMonitor.getLogMetadata())) { result.setLogMetadata(taskMonitor.getLogMetadata()); // assign final error message - DefaultTaskResultBuilder.assignErrorMessage(result, task); + DefaultTaskResultBuilder.assignErrorMessage(result, taskMonitor.getError()); taskMonitor.markLogMetaCollected(); log.info("Task log metadata collected, ji={}.", ji.getId()); } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java index a749f42a62..94f1f353fa 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.agent.runtime; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.schedule.JobIdentity; @@ -25,10 +25,10 @@ */ public interface TaskExecutor { - void execute(BaseTask task, JobContext jc); + void execute(Task task, JobContext jc); boolean cancel(JobIdentity ji); - BaseTask getTask(JobIdentity ji); + TaskRuntimeInfo getTaskRuntimeInfo(JobIdentity ji); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskMonitor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java similarity index 78% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskMonitor.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java index 37ce15218b..0507da329c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskMonitor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor; +package com.oceanbase.odc.agent.runtime; import java.util.HashMap; import java.util.Map; @@ -23,18 +23,26 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.lang3.StringUtils; +import com.google.common.annotations.VisibleForTesting; import com.oceanbase.odc.common.util.ObjectUtil; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.core.task.TaskThreadFactory; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.ExceptionListener; +import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.constants.JobAttributeKeyConstants; import com.oceanbase.odc.service.task.constants.JobConstants; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.constants.JobServerUrls; +import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.DefaultTaskResultBuilder; +import com.oceanbase.odc.service.task.executor.HeartbeatRequest; +import com.oceanbase.odc.service.task.executor.TaskReporter; +import com.oceanbase.odc.service.task.executor.TraceDecoratorThreadFactory; import com.oceanbase.odc.service.task.executor.logger.LogBizImpl; import com.oceanbase.odc.service.task.util.JobUtils; @@ -46,22 +54,26 @@ * @since 4.2.4 */ @Slf4j -public class TaskMonitor { +public class TaskMonitor implements ExceptionListener { private static final int REPORT_RESULT_RETRY_TIMES = Integer.MAX_VALUE; private static final long WAIT_AFTER_LOG_METADATA_COLLECT_MILLS = 5000L; private final TaskReporter reporter; - private final BaseTask task; + private final Task task; private final CloudObjectStorageService cloudObjectStorageService; private volatile long startTimeMilliSeconds; private ScheduledExecutorService reportScheduledExecutor; private ScheduledExecutorService heartScheduledExecutor; private Map logMetadata = new HashMap<>(); private AtomicLong logMetaCollectedMillis = new AtomicLong(0L); + // only save latest exception if any + // it will be cleaned if been fetched + protected AtomicReference latestException = new AtomicReference<>(); - public TaskMonitor(BaseTask task, CloudObjectStorageService cloudObjectStorageService) { + public TaskMonitor(Task task, TaskReporter taskReporter, + CloudObjectStorageService cloudObjectStorageService) { this.task = task; - this.reporter = new TaskReporter(task.getJobContext().getHostUrls());; + this.reporter = taskReporter; this.cloudObjectStorageService = cloudObjectStorageService; } @@ -134,7 +146,8 @@ private void destroy(ExecutorService executorService) { } } - private void reportTaskResult() { + @VisibleForTesting + protected void reportTaskResult() { if (JobUtils.isReportDisabled()) { return; } @@ -151,7 +164,8 @@ private void reportTaskResult() { copiedResult.getStatus(), String.format("%.2f", copiedResult.getProgress()), getTask().getTaskResult()); } - private boolean isTimeout() { + @VisibleForTesting + protected boolean isTimeout() { String milliSecStr = getTask().getJobContext().getJobParameters() .get(JobParametersKeyConstants.TASK_EXECUTION_TIMEOUT_MILLIS); @@ -167,7 +181,8 @@ private boolean isTimeout() { } } - private void doFinal() { + @VisibleForTesting + protected void doFinal() { DefaultTaskResult finalResult = DefaultTaskResultBuilder.build(getTask()); // Report final result @@ -188,16 +203,18 @@ private void doFinal() { if (JobUtils.isReportEnabled()) { // assign error for last report - DefaultTaskResultBuilder.assignErrorMessage(finalResult, getTask()); + DefaultTaskResultBuilder.assignErrorMessage(finalResult, getError()); // Report finish signal to task server - reportTaskResultWithRetry(finalResult, REPORT_RESULT_RETRY_TIMES); + reportTaskResultWithRetry(finalResult, REPORT_RESULT_RETRY_TIMES, + JobConstants.REPORT_TASK_INFO_INTERVAL_SECONDS); } else { waitForTaskResultPulled(); } log.info("Task id: {} exit.", getJobId()); } - private void waitForTaskResultPulled() { + @VisibleForTesting + protected void waitForTaskResultPulled() { long currentTimeMillis = System.currentTimeMillis(); while (this.logMetaCollectedMillis.get() == 0 || currentTimeMillis > this.logMetaCollectedMillis.get() + WAIT_AFTER_LOG_METADATA_COLLECT_MILLS) { @@ -213,21 +230,28 @@ private void waitForTaskResultPulled() { getJobId(), this.logMetaCollectedMillis.get()); } - private void uploadLogFileToCloudStorage(DefaultTaskResult finalResult) { + @VisibleForTesting + protected void uploadLogFileToCloudStorage(DefaultTaskResult finalResult) { Map logMap = finalResult.getLogMetadata(); + Map logMetaData = new HashMap<>(); + // append previous result + if (null != logMap) { + logMetaData.putAll(logMap); + } if (cloudObjectStorageService != null && cloudObjectStorageService.supported() && JobUtils.isK8sRunModeOfEnv()) { - logMap = new LogBizImpl().uploadLogFileToCloudStorage(finalResult.getJobIdentity(), - cloudObjectStorageService); + logMetaData.putAll(new LogBizImpl().uploadLogFileToCloudStorage(finalResult.getJobIdentity(), + cloudObjectStorageService)); } else { - logMap.put(JobAttributeKeyConstants.LOG_STORAGE_FAILED_REASON, + logMetaData.put(JobAttributeKeyConstants.LOG_STORAGE_FAILED_REASON, "cloudObjectStorageService is null or not supported"); } - finalResult.setLogMetadata(logMap); + finalResult.setLogMetadata(logMetaData); } - private void reportTaskResultWithRetry(DefaultTaskResult result, int retries) { + @VisibleForTesting + protected boolean reportTaskResultWithRetry(DefaultTaskResult result, int retries, int retryIntervalSeconds) { if (result.getStatus() == TaskStatus.DONE) { result.setProgress(100.0); } @@ -237,16 +261,17 @@ private void reportTaskResultWithRetry(DefaultTaskResult result, int retries) { boolean success = reporter.report(JobServerUrls.TASK_UPLOAD_RESULT, result); if (success) { log.info("Report task result successfully"); - break; + return true; } else { log.warn("Report task result failed, will retry after {} seconds, remaining retries: {}", - JobConstants.REPORT_TASK_INFO_INTERVAL_SECONDS, retries - retryTimes); - Thread.sleep(JobConstants.REPORT_TASK_INFO_INTERVAL_SECONDS * 1000L); + retryIntervalSeconds, retries - retryTimes); + Thread.sleep(retryIntervalSeconds * 1000L); } } catch (Throwable e) { log.warn("Report task result failed, taskId: {}", getJobId(), e); } } + return false; } private HeartbeatRequest buildHeartRequest() { @@ -256,7 +281,7 @@ private HeartbeatRequest buildHeartRequest() { return request; } - private BaseTask getTask() { + private Task getTask() { return task; } @@ -267,4 +292,15 @@ private TaskReporter getReporter() { private Long getJobId() { return getTask().getJobContext().getJobIdentity().getId(); } + + public Throwable getError() { + Throwable e = latestException.getAndSet(null); + log.info("retrieve exception = {}", null == e ? null : e.getMessage()); + return e; + } + + public void onException(Throwable e) { + log.info("found exception", e); + this.latestException.set(e); + } } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskRuntimeInfo.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskRuntimeInfo.java new file mode 100644 index 0000000000..56483ee6cf --- /dev/null +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskRuntimeInfo.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.agent.runtime; + +import java.util.concurrent.Future; + +import com.oceanbase.odc.service.task.Task; + +import lombok.Data; + +/** + * @author longpeng.zlp + * @date 2024/10/24 14:32 + */ +@Data +public class TaskRuntimeInfo { + private final Task task; + private final Future future; + private final TaskMonitor taskMonitor; +} diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java index f27f29f0dd..f6c24f8a2d 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java @@ -15,8 +15,9 @@ */ package com.oceanbase.odc.agent.runtime; -import java.util.HashMap; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -27,13 +28,18 @@ import com.oceanbase.odc.common.concurrent.ExecutorUtils; import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.task.TaskThreadFactory; +import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; +import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; import com.oceanbase.odc.service.task.ExceptionListener; import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.TaskEventListener; import com.oceanbase.odc.service.task.caller.JobContext; +import com.oceanbase.odc.service.task.executor.TaskReporter; import com.oceanbase.odc.service.task.executor.TraceDecoratorThreadFactory; import com.oceanbase.odc.service.task.schedule.JobIdentity; +import com.oceanbase.odc.service.task.util.CloudObjectStorageServiceBuilder; +import com.oceanbase.odc.service.task.util.JobUtils; import lombok.extern.slf4j.Slf4j; @@ -47,8 +53,7 @@ public class ThreadPoolTaskExecutor implements TaskExecutor { private static final TaskExecutor TASK_EXECUTOR = new ThreadPoolTaskExecutor(); - private final Map> tasks = new HashMap<>(); - private final Map> futures = new HashMap<>(); + private final Map tasks = new ConcurrentHashMap<>(); private final ExecutorService executor; private ThreadPoolTaskExecutor() { @@ -61,38 +66,97 @@ public static TaskExecutor getInstance() { } @Override - synchronized public void execute(BaseTask task, JobContext jc) { + synchronized public void execute(Task task, JobContext jc) { JobIdentity jobIdentity = jc.getJobIdentity(); log.info("Start to execute task, jobIdentity={}.", jobIdentity.getId()); if (tasks.containsKey(jobIdentity)) { throw new IllegalArgumentException("Task already exists, jobIdentity=" + jobIdentity.getId()); } + // init cloud objet storage service and task monitor + CloudObjectStorageService cloudObjectStorageService = buildCloudStorageService(jc); + TaskMonitor taskMonitor = new TaskMonitor(task, new TaskReporter(jc.getHostUrls()), cloudObjectStorageService); + Future future = executor.submit(() -> { try { task.start(new TaskContext() { @Override public ExceptionListener getExceptionListener() { - return task; + return taskMonitor; } @Override public JobContext getJobContext() { return jc; } + + @Override + public TaskEventListener getTaskEventListener() { + return taskEventListener(taskMonitor); + } + + @Override + public CloudObjectStorageService getSharedStorage() { + return cloudObjectStorageService; + } }); } catch (Exception e) { log.error("Task start failed, jobIdentity={}.", jobIdentity.getId(), e); - task.onException(e); + taskMonitor.onException(e); } }); - futures.put(jobIdentity, future); - tasks.put(jobIdentity, task); + tasks.put(jobIdentity, new TaskRuntimeInfo(task, future, taskMonitor)); + } + + /** + * build task event listener + * + * @param taskMonitor + * @return + */ + private TaskEventListener taskEventListener(TaskMonitor taskMonitor) { + return new TaskEventListener() { + @Override + public void onTaskStart(Task task) { + taskMonitor.monitor(); + } + + @Override + public void onTaskStop(Task task) {} + + @Override + public void onTaskModify(Task task) {} + + @Override + public void onTaskFinalize(Task task) { + taskMonitor.finalWork(); + } + }; + } + + /** + * build task monitor + * + * @param jobContext + * @return + */ + protected CloudObjectStorageService buildCloudStorageService(JobContext jobContext) { + Optional storageConfig = JobUtils.getObjectStorageConfiguration(); + CloudObjectStorageService cloudObjectStorageService = null; + try { + if (storageConfig.isPresent()) { + cloudObjectStorageService = CloudObjectStorageServiceBuilder.build(storageConfig.get()); + } + } catch (Throwable e) { + log.warn("Init cloud object storage service failed, id={}.", jobContext.getJobIdentity().getId(), e); + } + return cloudObjectStorageService; } @Override public boolean cancel(JobIdentity ji) { - Task task = getTask(ji); + TaskRuntimeInfo runtimeInfo = getTaskRuntimeInfo(ji); + Task task = runtimeInfo.getTask(); Future stopFuture = executor.submit(task::stop); boolean result = false; try { @@ -117,9 +181,9 @@ public boolean cancel(JobIdentity ji) { } @Override - public BaseTask getTask(JobIdentity ji) { - BaseTask task = tasks.get(ji); - PreConditions.notNull(task, "task", "Task not found, jobIdentity=" + ji.getId()); - return task; + public TaskRuntimeInfo getTaskRuntimeInfo(JobIdentity ji) { + TaskRuntimeInfo runtimeInfo = tasks.get(ji); + PreConditions.notNull(runtimeInfo, "task", "Task not found, jobIdentity=" + ji.getId()); + return runtimeInfo; } } diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java new file mode 100644 index 0000000000..3e52e15c4b --- /dev/null +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.agent.runtime; + +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; +import com.oceanbase.odc.service.task.Task; +import com.oceanbase.odc.service.task.TaskContext; +import com.oceanbase.odc.service.task.caller.JobContext; +import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.TaskReporter; + +/** + * @author longpeng.zlp + * @date 2024/11/7 17:45 + */ +public class TaskMonitorTest { + @Test + public void testTaskMonitorOnException() { + TaskMonitor taskMonitor = new TaskMonitor(new MockBaseTask(), Mockito.mock(TaskReporter.class), Mockito.mock( + CloudObjectStorageService.class)); + Assert.assertNull(taskMonitor.getError()); + taskMonitor.onException(new Throwable("error")); + Throwable ex = taskMonitor.getError(); + Assert.assertEquals(ex.getMessage(), "error"); + Assert.assertNull(taskMonitor.getError()); + } + + @Test + public void testTaskMonitorReportRetryFailed() { + TaskReporter taskReporter = Mockito.mock(TaskReporter.class); + Mockito.when(taskReporter.report(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); + TaskMonitor taskMonitor = new TaskMonitor(new MockBaseTask(), taskReporter, Mockito.mock( + CloudObjectStorageService.class)); + Assert.assertFalse(taskMonitor.reportTaskResultWithRetry(new DefaultTaskResult(), 3, 1)); + } + + @Test + public void testTaskMonitorReportRetrySuccess() { + TaskReporter taskReporter = Mockito.mock(TaskReporter.class); + Mockito.when(taskReporter.report(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(true); + TaskMonitor taskMonitor = new TaskMonitor(new MockBaseTask(), taskReporter, Mockito.mock( + CloudObjectStorageService.class)); + Assert.assertTrue(taskMonitor.reportTaskResultWithRetry(new DefaultTaskResult(), 3, 1)); + } + + private static final class MockBaseTask implements Task { + + @Override + public void start(TaskContext taskContext) {} + + @Override + public boolean stop() { + return false; + } + + @Override + public boolean modify(Map jobParameters) { + return false; + } + + @Override + public double getProgress() { + return 0; + } + + @Override + public JobContext getJobContext() { + return null; + } + + @Override + public TaskStatus getStatus() { + return TaskStatus.RUNNING; + } + + @Override + public String getTaskResult() { + return "result"; + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskContext.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskContext.java index 7c284ecbfd..497a3b703b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskContext.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskContext.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.task; +import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; import com.oceanbase.odc.service.task.caller.JobContext; /** @@ -37,4 +38,18 @@ public interface TaskContext { * @return */ JobContext getJobContext(); + + /** + * provide task event listener + * + * @return + */ + TaskEventListener getTaskEventListener(); + + /** + * get shared storage for task upload or download file + * + * @return + */ + CloudObjectStorageService getSharedStorage(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskEventListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskEventListener.java new file mode 100644 index 0000000000..e68a2d8c97 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskEventListener.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task; + +/** + * listen task event + * + * @author longpeng.zlp + * @date 2024/10/24 11:51 + */ +public interface TaskEventListener { + /** + * call when task start called + * + * @param task + */ + void onTaskStart(Task task); + + + /** + * call when task stop called + * + * @param task + */ + void onTaskStop(Task task); + + + /** + * call when task modify called + * + * @param task + */ + void onTaskModify(Task task); + + /** + * call when task final closed + * + * @param task + */ + void onTaskFinalize(Task task); + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java index 9193ec028c..a63a753181 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java @@ -15,26 +15,19 @@ */ package com.oceanbase.odc.service.task.base; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import com.oceanbase.odc.core.shared.constant.TaskStatus; -import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; -import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; -import com.oceanbase.odc.service.task.ExceptionListener; import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.TaskContext; import com.oceanbase.odc.service.task.caller.DefaultJobContext; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.executor.TaskMonitor; -import com.oceanbase.odc.service.task.util.CloudObjectStorageServiceBuilder; -import com.oceanbase.odc.service.task.util.JobUtils; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; /** @@ -42,40 +35,26 @@ * @date 2023/11/22 20:16 */ @Slf4j -public abstract class BaseTask implements Task, ExceptionListener { +public abstract class BaseTask implements Task { private final AtomicBoolean closed = new AtomicBoolean(false); - private JobContext context; - private Map jobParameters; + protected TaskContext context; + private DefaultJobContext jobContext; private volatile TaskStatus status = TaskStatus.PREPARING; - private CloudObjectStorageService cloudObjectStorageService; - // only save latest exception if any - // it will be cleaned if been fetched - protected AtomicReference latestException = new AtomicReference<>(); - @Getter - private TaskMonitor taskMonitor; @Override public void start(TaskContext taskContext) { - this.context = taskContext.getJobContext(); - log.info("Start task, id={}.", context.getJobIdentity().getId()); + this.context = taskContext; + jobContext = copyJobContext(taskContext); + log.info("Start task, id={}.", jobContext.getJobIdentity().getId()); + log.info("Init task parameters success, id={}.", jobContext.getJobIdentity().getId()); - this.jobParameters = Collections.unmodifiableMap(context.getJobParameters()); - log.info("Init task parameters success, id={}.", context.getJobIdentity().getId()); - - try { - initCloudObjectStorageService(); - } catch (Exception e) { - log.warn("Init cloud object storage service failed, id={}.", getJobId(), e); - } - - this.taskMonitor = createTaskMonitor(); try { - doInit(context); + doInit(jobContext); updateStatus(TaskStatus.RUNNING); - taskMonitor.monitor(); - if (doStart(context, taskContext)) { + context.getTaskEventListener().onTaskStart(this); + if (doStart(jobContext, taskContext)) { updateStatus(TaskStatus.DONE); } else { updateStatus(TaskStatus.FAILED); @@ -89,10 +68,6 @@ public void start(TaskContext taskContext) { } } - protected TaskMonitor createTaskMonitor() { - return new TaskMonitor(this, cloudObjectStorageService); - } - @Override public boolean stop() { try { @@ -122,9 +97,7 @@ public boolean modify(Map jobParameters) { log.warn("Task is already finished, cannot modify parameters, id={}", getJobId()); return false; } - DefaultJobContext ctx = (DefaultJobContext) getJobContext(); - ctx.setJobParameters(jobParameters); - this.jobParameters = Collections.unmodifiableMap(jobParameters); + jobContext.setJobParameters(Collections.unmodifiableMap(jobParameters)); try { afterModifiedJobParameters(); } catch (Exception e) { @@ -133,10 +106,6 @@ public boolean modify(Map jobParameters) { return true; } - private void initCloudObjectStorageService() { - Optional storageConfig = JobUtils.getObjectStorageConfiguration(); - storageConfig.ifPresent(osc -> this.cloudObjectStorageService = CloudObjectStorageServiceBuilder.build(osc)); - } private void close() { if (closed.compareAndSet(false, true)) { @@ -146,14 +115,12 @@ private void close() { // do nothing } log.info("Task completed, id={}, status={}.", getJobId(), getStatus()); - taskMonitor.finalWork(); + if (null != context) { + context.getTaskEventListener().onTaskFinalize(this); + } } } - protected CloudObjectStorageService getCloudObjectStorageService() { - return cloudObjectStorageService; - } - @Override public TaskStatus getStatus() { return status; @@ -161,7 +128,7 @@ public TaskStatus getStatus() { @Override public JobContext getJobContext() { - return context; + return jobContext; } protected void updateStatus(TaskStatus status) { @@ -170,7 +137,7 @@ protected void updateStatus(TaskStatus status) { } protected Map getJobParameters() { - return this.jobParameters; + return jobContext.getJobParameters(); } private Long getJobId() { @@ -197,14 +164,21 @@ protected void afterModifiedJobParameters() throws Exception { // do nothing } - public Throwable getError() { - Throwable e = latestException.getAndSet(null); - log.info("retrieve exception = {}", null == e ? null : e.getMessage()); - return e; - } - - public void onException(Throwable e) { - log.info("found exception", e); - this.latestException.set(e); + // deep copy job context + protected DefaultJobContext copyJobContext(TaskContext taskContext) { + DefaultJobContext ret = new DefaultJobContext(); + JobContext src = taskContext.getJobContext(); + ret.setJobIdentity(src.getJobIdentity()); + if (null != src.getJobProperties()) { + ret.setJobProperties(new HashMap<>(src.getJobProperties())); + } + if (null != src.getJobParameters()) { + ret.setJobParameters(Collections.unmodifiableMap(new HashMap<>(src.getJobParameters()))); + } + ret.setJobClass(src.getJobClass()); + if (null != src.getHostUrls()) { + ret.setHostUrls(new ArrayList<>(src.getHostUrls())); + } + return ret; } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java index cdc9c7e6d6..56d0a504cf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java @@ -141,7 +141,7 @@ public class DatabaseChangeTask extends BaseTask { private long taskId; @Override - protected void doInit(JobContext context) { + protected void doInit(JobContext jobContext) { taskId = getJobContext().getJobIdentity().getId(); log.info("Initiating database change task, taskId={}", taskId); this.parameters = JobUtils.fromJson(getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), @@ -162,7 +162,7 @@ protected void doInit(JobContext context) { try { SizeAwareInputStream sizeAwareInputStream = ObjectStorageUtils.loadObjectsForTask(this.parameters.getSqlFileObjectMetadatas(), - getCloudObjectStorageService(), JobUtils.getExecutorDataPath(), -1); + this.context.getSharedStorage(), JobUtils.getExecutorDataPath(), -1); sqlTotalBytes += sizeAwareInputStream.getTotalBytes(); sqlInputStream = sizeAwareInputStream.getInputStream(); } catch (IOException exception) { @@ -429,7 +429,7 @@ private void writeZipFile() { OdcFileUtil.zip(String.format(zipFileRootPath), String.format("%s.zip", zipFileRootPath)); log.info("Database change task result set was saved as local zip file, file name={}", zipFileId); // Public cloud scenario, need to upload files to OSS - CloudObjectStorageService cloudObjectStorageService = getCloudObjectStorageService(); + CloudObjectStorageService cloudObjectStorageService = context.getSharedStorage(); if (Objects.nonNull(cloudObjectStorageService) && cloudObjectStorageService.supported()) { File tempZipFile = new File(String.format("%s.zip", zipFileRootPath)); try { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java index 2db307a73a..cee3873bb0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java @@ -197,7 +197,7 @@ private void loadUploadFileInputStream() throws IOException { List objectMetadataList = this.parameters.getSqlFileObjectMetadatas(); if (Objects.nonNull(params) && CollectionUtils.isNotEmpty(objectMetadataList)) { this.uploadFileInputStream = ObjectStorageUtils.loadObjectsForTask(objectMetadataList, - getCloudObjectStorageService(), JobUtils.getExecutorDataPath(), -1).getInputStream(); + context.getSharedStorage(), JobUtils.getExecutorDataPath(), -1).getInputStream(); this.uploadFileSqlIterator = SqlUtils.iterator(this.parameters.getConnectionConfig().getDialectType(), params.getDelimiter(), this.uploadFileInputStream, StandardCharsets.UTF_8); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java index 8bf4001980..dac0e87044 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java @@ -200,7 +200,7 @@ private void loadUploadFileInputStream() throws IOException { if (CollectionUtils.isNotEmpty(objectMetadataList)) { this.uploadFileInputStream = ObjectStorageUtils - .loadObjectsForTask(objectMetadataList, getCloudObjectStorageService(), + .loadObjectsForTask(objectMetadataList, context.getSharedStorage(), JobUtils.getExecutorDataPath(), parameters.getRollbackProperties().getMaxRollbackContentSizeBytes()) .getInputStream(); @@ -223,7 +223,7 @@ private void handleRollbackResult(String rollbackResult) { String resultFileId = StringUtils.uuid(); String filePath = String.format("%s/%s.sql", resultFileRootPath, resultFileId); FileUtils.writeStringToFile(new File(filePath), rollbackResult, StandardCharsets.UTF_8); - CloudObjectStorageService cloudObjectStorageService = getCloudObjectStorageService(); + CloudObjectStorageService cloudObjectStorageService = context.getSharedStorage(); if (Objects.nonNull(cloudObjectStorageService) && cloudObjectStorageService.supported()) { File tempFile = new File(filePath); try { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java index ead4aed2fe..5d3ea7630f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java @@ -215,7 +215,7 @@ private void initSqlInputStream() { return; } - CloudObjectStorageService cloudObjectStorageService = getCloudObjectStorageService(); + CloudObjectStorageService cloudObjectStorageService = context.getSharedStorage(); if (Objects.isNull(cloudObjectStorageService) || !cloudObjectStorageService.supported()) { log.warn("Cloud object storage service not supported."); throw new UnexpectedException("Cloud object storage service not supported"); @@ -468,7 +468,7 @@ private void upload() { private String uploadToOSS(String filePath) { // Public cloud scenario, need to upload files to OSS - CloudObjectStorageService cloudObjectStorageService = getCloudObjectStorageService(); + CloudObjectStorageService cloudObjectStorageService = context.getSharedStorage(); if (Objects.nonNull(cloudObjectStorageService) && cloudObjectStorageService.supported()) { File file = new File(filePath); String ossAddress; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResultBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResultBuilder.java index b17d15d0f6..e70a0256be 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResultBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResultBuilder.java @@ -16,7 +16,7 @@ package com.oceanbase.odc.service.task.executor; import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.util.JobUtils; /** @@ -26,7 +26,7 @@ */ public class DefaultTaskResultBuilder { - public static DefaultTaskResult build(BaseTask task) { + public static DefaultTaskResult build(Task task) { DefaultTaskResult result = new DefaultTaskResult(); result.setResultJson(JsonUtils.toJson(task.getTaskResult())); result.setStatus(task.getStatus()); @@ -36,8 +36,7 @@ public static DefaultTaskResult build(BaseTask task) { return result; } - public static void assignErrorMessage(DefaultTaskResult result, BaseTask task) { - Throwable e = task.getError(); + public static void assignErrorMessage(DefaultTaskResult result, Throwable e) { result.setErrorMessage(null == e ? null : e.getMessage()); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyResourceJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyResourceJob.java index ae76e73713..36075d4746 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyResourceJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyResourceJob.java @@ -83,6 +83,7 @@ private void destroyResource(ResourceEntity resourceEntity) { Map eventMessage = AlarmUtils.createAlarmMapBuilder() .item(AlarmUtils.ORGANIZATION_NAME, AlarmUtils.ODC_RESOURCE) .item(AlarmUtils.RESOURCE_ID_NAME, String.valueOf(resourceEntity.getId())) + .item(AlarmUtils.RESOURCE_TYPE, resourceEntity.getResourceType()) .item(AlarmUtils.MESSAGE_NAME, MessageFormat.format("Job resource destroy failed, resourceID={0}, message={1}", resourceEntity.getId(), e.getMessage())) @@ -95,7 +96,6 @@ private void destroyResource(ResourceEntity resourceEntity) { }); } - private JobConfiguration getConfiguration() { return configuration; } diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java index a60dfeb544..1e79e8b7ab 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java @@ -24,15 +24,16 @@ import org.mockito.Mockito; import com.oceanbase.odc.common.util.SystemUtils; +import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; import com.oceanbase.odc.service.task.ExceptionListener; import com.oceanbase.odc.service.task.TaskContext; +import com.oceanbase.odc.service.task.TaskEventListener; import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.caller.DefaultJobContext; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; import com.oceanbase.odc.service.task.executor.DefaultTaskResult; import com.oceanbase.odc.service.task.executor.DefaultTaskResultBuilder; -import com.oceanbase.odc.service.task.executor.TaskMonitor; import com.oceanbase.odc.service.task.schedule.JobIdentity; /** @@ -61,19 +62,30 @@ public void testExceptionListenerNormal() { SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_EXECUTOR_PORT); }).thenReturn("9099"); DummyBaseTask dummyBaseTask = new DummyBaseTask(false); + DummyErrorListener dummyErrorListener = new DummyErrorListener(); dummyBaseTask.start(new TaskContext() { @Override public ExceptionListener getExceptionListener() { - return dummyBaseTask; + return dummyErrorListener; } @Override public JobContext getJobContext() { return jobContext; } + + @Override + public TaskEventListener getTaskEventListener() { + return Mockito.mock(TaskEventListener.class); + } + + @Override + public CloudObjectStorageService getSharedStorage() { + return Mockito.mock(CloudObjectStorageService.class); + } }); DefaultTaskResult taskResult = DefaultTaskResultBuilder.build(dummyBaseTask); - DefaultTaskResultBuilder.assignErrorMessage(taskResult, dummyBaseTask); + DefaultTaskResultBuilder.assignErrorMessage(taskResult, dummyErrorListener.error); Assert.assertNull(taskResult.getErrorMessage()); } } @@ -85,23 +97,43 @@ public void testExceptionListenerWithException() { SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_EXECUTOR_PORT); }).thenReturn("9099"); DummyBaseTask dummyBaseTask = new DummyBaseTask(true); + DummyErrorListener dummyErrorListener = new DummyErrorListener(); dummyBaseTask.start(new TaskContext() { @Override public ExceptionListener getExceptionListener() { - return dummyBaseTask; + return dummyErrorListener; } @Override public JobContext getJobContext() { return jobContext; } + + @Override + public TaskEventListener getTaskEventListener() { + return Mockito.mock(TaskEventListener.class); + } + + @Override + public CloudObjectStorageService getSharedStorage() { + return Mockito.mock(CloudObjectStorageService.class); + } }); DefaultTaskResult taskResult = DefaultTaskResultBuilder.build(dummyBaseTask); - DefaultTaskResultBuilder.assignErrorMessage(taskResult, dummyBaseTask); + DefaultTaskResultBuilder.assignErrorMessage(taskResult, dummyErrorListener.error); Assert.assertEquals(taskResult.getErrorMessage(), "exception should be thrown"); } } + private static final class DummyErrorListener implements ExceptionListener { + private Throwable error; + + @Override + public void onException(Throwable e) { + this.error = e; + } + } + private static final class DummyBaseTask extends BaseTask { private final boolean shouldThrowException; @@ -120,10 +152,6 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex return true; } - protected TaskMonitor createTaskMonitor() { - return Mockito.mock(TaskMonitor.class); - } - @Override protected void doStop() throws Exception {} From 873cc460f82a72bfc2a9421df752a7b108432f3d Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Fri, 8 Nov 2024 11:51:27 +0800 Subject: [PATCH 023/118] feat(ticket): allow project members to view and owners to update all tickets in the project (#3739) * add project in detail flow instance * listFlowInstances refactor * refactor * delete unused code * fix pmd * remove not implemented code * fix schedule * fix schedule * fix ut * fix ut * fix apply project * fix --- .../service/flow/FlowInstanceServiceTest.java | 15 +- .../flow/FlowResponseMapperFactoryTest.java | 2 + .../controller/v2/FlowInstanceController.java | 8 +- .../controller/v2/ScheduleControllerHist.java | 4 +- .../odc/config/HookConfiguration.java | 2 +- .../odc/metadb/flow/FlowInstanceSpecs.java | 4 +- .../UserResourceRoleRepository.java | 6 +- .../collaboration/project/ProjectService.java | 5 + .../odc/service/flow/FlowInstanceService.java | 166 +++++++----------- .../service/flow/FlowPermissionHelper.java | 106 +++++++++++ .../flow/FlowTaskInstanceLoggerService.java | 11 +- .../service/flow/FlowTaskInstanceService.java | 73 +++++--- .../factory/FlowResponseMapperFactory.java | 84 +++++---- .../flow/model/FlowInstanceDetailResp.java | 4 + .../flow/model/QueryFlowInstanceParams.java | 3 +- .../odc/service/iam/ResourceRoleService.java | 8 +- .../odc/service/iam/UserService.java | 4 + .../odc/service/schedule/ScheduleService.java | 13 +- .../ScheduleResponseMapperFactory.java | 9 + .../schedule/model/ScheduleOverviewHist.java | 3 + .../ShadowTableComparingService.java | 3 +- .../StructureComparisonService.java | 3 +- 22 files changed, 339 insertions(+), 197 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowPermissionHelper.java diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java index 4c6bf80d5d..1242ca481e 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java @@ -60,6 +60,7 @@ import com.oceanbase.odc.core.shared.constant.ResourceRoleName; import com.oceanbase.odc.core.shared.constant.TaskErrorStrategy; import com.oceanbase.odc.core.shared.constant.TaskType; +import com.oceanbase.odc.core.shared.exception.AccessDeniedException; import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.core.shared.exception.OverLimitException; import com.oceanbase.odc.metadb.flow.FlowInstanceEntity; @@ -76,6 +77,7 @@ import com.oceanbase.odc.metadb.task.TaskRepository; import com.oceanbase.odc.plugin.task.api.datatransfer.model.DataTransferConfig; import com.oceanbase.odc.plugin.task.api.datatransfer.model.DataTransferObject; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; @@ -206,6 +208,9 @@ public void setUp() { UserEntity user = new UserEntity(); user.setEnabled(true); when(userService.nullSafeGet(anyLong())).thenReturn(user); + when(userService.getCurrentUserJoinedProjectIds()).thenReturn(Collections.singleton(1L)); + when(databaseService.listDatabasesByIds(Mockito.anyCollection())) + .thenReturn(Collections.singletonList(getDatabase())); } @Test @@ -401,8 +406,7 @@ public void detail_unauthorized_expThrown() { entity.setCreatorId(-1); flowInstanceRepository.saveAndFlush(entity); - thrown.expectMessage(String.format("ODC_FLOW_INSTANCE not found by id=%d", flowInstance.getId())); - thrown.expect(NotFoundException.class); + thrown.expect(AccessDeniedException.class); flowInstanceService.detail(flowInstance.getId()); } @@ -624,9 +628,16 @@ private Database getDatabase() { ConnectionConfig connectionConfig = new ConnectionConfig(); connectionConfig.setId(1L); database.setDataSource(connectionConfig); + database.setProject(getProject()); return database; } + private Project getProject() { + Project project = new Project(); + project.setId(1L); + return project; + } + private List getApprovalNodes() { List nodes = new ArrayList<>(); ApprovalNodeConfig first = new ApprovalNodeConfig(); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java index f44e2e90c9..7f2b058682 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java @@ -52,6 +52,7 @@ import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.metadb.task.TaskRepository; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; @@ -151,6 +152,7 @@ public void generateFlowInstanceDetailResp_entityInput_returnDesp() { id -> Collections.singletonList(TestRandom.nextObject(Date.class))) .getCandidatesByFlowInstanceId( id -> Collections.singleton(TestRandom.nextObject(UserEntity.class))) + .getProjectById(id -> TestRandom.nextObject(Project.class)) .build(); FlowInstanceDetailResp detailResp = mapper.map(instanceEntity); diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java index b98c5b7be0..e9f2a0e340 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java @@ -118,7 +118,7 @@ public PaginatedResponse listFlowInstances( @RequestParam(name = "approveByCurrentUser") Boolean approveByCurrentUser, @RequestParam(required = false, name = "containsAll", defaultValue = "false") Boolean containsAll, @RequestParam(required = false, name = "parentInstanceId") Long parentInstanceId, - @RequestParam(required = false, name = "projectId") Long projectId) { + @RequestParam(required = false, name = "projectId") Set projectIds) { QueryFlowInstanceParams params = QueryFlowInstanceParams.builder() .connectionIds(connectionIds) .id(fuzzySearchKeyword) @@ -132,7 +132,7 @@ public PaginatedResponse listFlowInstances( .approveByCurrentUser(approveByCurrentUser) .containsAll(containsAll) .parentInstanceId(parentInstanceId) - .projectId(projectId) + .projectIds(projectIds) .build(); return Responses.paginated(flowInstanceService.list(pageable, params)); } @@ -199,14 +199,14 @@ public SuccessResponse getMetaInfo() { @ApiOperation(value = "getResult", notes = "获取任务结果") @RequestMapping(value = "/{id:[\\d]+}/tasks/result", method = RequestMethod.GET) public ListResponse getResult(@PathVariable Long id) throws IOException { - return Responses.list(flowTaskInstanceService.getResult(id, false)); + return Responses.list(flowTaskInstanceService.getResult(id)); } @ApiOperation(value = "getResult", notes = "获取任务结果") @RequestMapping(value = "/{flowInstanceId:[\\d]+}/tasks/{nodeInstanceId:[\\d]+}/result", method = RequestMethod.GET) public ListResponse getResult( @PathVariable Long flowInstanceId, @PathVariable Long nodeInstanceId) throws IOException { - return Responses.list(flowTaskInstanceService.getResult(flowInstanceId, nodeInstanceId, false)); + return Responses.list(flowTaskInstanceService.getResult(flowInstanceId, nodeInstanceId)); } @ApiOperation(value = "download", notes = "下载任务数据，仅用于 模拟数据、导出、数据库变更 任务") diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java index 0b372eadf5..a757ec92de 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java @@ -71,7 +71,7 @@ public PaginatedResponse list( @RequestParam(required = false, name = "startTime") Date startTime, @RequestParam(required = false, name = "endTime") Date endTime, @RequestParam(required = false, name = "creator") String creator, - @RequestParam(required = false, name = "projectId") Long projectId) { + @RequestParam(required = false, name = "projectId") Set projectIds) { QueryScheduleParams req = QueryScheduleParams.builder() .id(id) @@ -81,7 +81,7 @@ public PaginatedResponse list( .startTime(startTime) .endTime(endTime) .creator(creator) - .projectId(projectId) + .projectIds(projectIds) .build(); return Responses.paginated(scheduleService.list(pageable, req)); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/config/HookConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/config/HookConfiguration.java index a2d6ff048e..57a715a17a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/config/HookConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/config/HookConfiguration.java @@ -147,7 +147,7 @@ private void projectReferenceCheck(Long userId, Long organizationId) { .collect(Collectors.toMap(ProjectEntity::getId, p -> p)); Map> projectId2ResourceRoleNames = - resourceRoleService.getProjectId2ResourceRoleNames(userId).entrySet().stream() + resourceRoleService.getProjectId2ResourceRoleNames(userId, organizationId).entrySet().stream() .filter(e -> e.getValue().contains(ResourceRoleName.OWNER) || e.getValue().contains(ResourceRoleName.DBA)) .filter(e -> id2Project.containsKey(e.getKey())) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/flow/FlowInstanceSpecs.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/flow/FlowInstanceSpecs.java index ed8d27727b..dbaa041185 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/flow/FlowInstanceSpecs.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/flow/FlowInstanceSpecs.java @@ -98,7 +98,7 @@ public static Specification parentInstanceIdIn(Collection projectIdEquals(Long projectId) { - return SpecificationUtil.columnEqual(FLOW_INSTANCE_PROJECT_ID, projectId); + public static Specification projectIdIn(Collection projectIds) { + return SpecificationUtil.columnIn(FLOW_INSTANCE_PROJECT_ID, projectIds); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java index cfeb1a7437..f6d9b96c28 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java @@ -103,10 +103,10 @@ List findByResourceIdAndTypeAndName(@Param("resourceId") @Param("resourceType") ResourceType resourceType, @Param("roleName") String roleName); @Query(value = "select i_urr.* from iam_user_resource_role i_urr inner join iam_resource_role i_rr on i_urr.resource_role_id = i_rr.id " - + "where i_rr.resource_type = :#{#resourceType.name()} and i_urr.user_id = :userId", + + "where i_rr.resource_type = :#{#resourceType.name()} and i_urr.user_id = :userId and i_urr.organization_id = :organizationId", nativeQuery = true) - List findByUserIdAndResourceType(@Param("userId") Long userId, - @Param("resourceType") ResourceType resourceType); + List findByUserIdAndResourceTypeAndOrganizationId(@Param("userId") Long userId, + @Param("resourceType") ResourceType resourceType, @Param("organizationId") Long organizationId); default List batchCreate(List entities) { String sql = InsertSqlTemplateBuilder.from("iam_user_resource_role") diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java index 25e09b1f06..d41d93fff4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java @@ -323,6 +323,11 @@ public Page list(@Valid QueryProjectParams params, @NotNull Pageable pa }); } + @SkipAuthorize("odc internal usage") + public List listByIds(@NotEmpty Set ids) { + return repository.findAllById(ids).stream().map(projectMapper::entityToModel).collect(Collectors.toList()); + } + private Page innerList(@Valid QueryProjectParams params, @NotNull Pageable pageable, @NotNull Predicate predicate) { List userResourceRoles = diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java index 8b595d28be..29c728fbc2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java @@ -262,6 +262,8 @@ public class FlowInstanceService { @Autowired private EnvironmentService environmentService; @Autowired + private FlowPermissionHelper flowPermissionHelper; + @Autowired private MeterManager meterManager; private static final long MAX_EXPORT_OBJECT_COUNT = 10000; @@ -413,8 +415,8 @@ public FlowMetaInfo getMetaInfo() { } public Page listAll(@NotNull Pageable pageable, @NotNull QueryFlowInstanceParams params) { - if (Objects.nonNull(params.getProjectId())) { - projectPermissionValidator.checkProjectRole(params.getProjectId(), ResourceRoleName.all()); + if (Objects.nonNull(params.getProjectIds())) { + projectPermissionValidator.checkProjectRole(params.getProjectIds(), ResourceRoleName.all()); } if (params.getParentInstanceId() != null) { // TODO 4.1.3 自动运行模块改造完成后剥离 @@ -438,9 +440,8 @@ public Page listAll(@NotNull Pageable pageable, @NotNull Que Specification specification = Specification.where(FlowInstanceSpecs.idIn(flowInstanceIds)) .and(FlowInstanceSpecs.organizationIdEquals(authenticationFacade.currentOrganizationId())); - // TODO Remove the checker after the SQL console development is completed - if (params.getProjectId() != null) { - specification = specification.and(FlowInstanceSpecs.projectIdEquals(params.getProjectId())); + if (CollectionUtils.isNotEmpty(params.getProjectIds())) { + specification = specification.and(FlowInstanceSpecs.projectIdIn(params.getProjectIds())); } return flowInstanceRepository.findAll(specification, pageable); } @@ -488,75 +489,36 @@ public Page listAll(@NotNull Pageable pageable, @NotNull Que specification = specification.and(FlowInstanceViewSpecs.taskTypeIn(types)); } - Set resourceRoleIdentifiers = userService.getCurrentUserResourceRoleIdentifiers(); - if (params.getContainsAll()) { - // does not join any project - if (CollectionUtils.isEmpty(resourceRoleIdentifiers)) { - specification = - specification.and(FlowInstanceViewSpecs.creatorIdEquals(authenticationFacade.currentUserId())); - return flowInstanceViewRepository.findAll(specification, pageable).map(FlowInstanceEntity::from); - } - // find by project id - if (Objects.nonNull(params.getProjectId())) { - specification = specification.and(FlowInstanceViewSpecs.projectIdEquals(params.getProjectId())); - // if other project roles, show current user's created, waiting for approval and approved/rejected - // tickets - if (!projectPermissionValidator.hasProjectRole(params.getProjectId(), - Collections.singletonList(ResourceRoleName.OWNER))) { - specification = specification.and(FlowInstanceViewSpecs.leftJoinFlowInstanceApprovalView( - resourceRoleIdentifiers, authenticationFacade.currentUserId(), - FlowNodeStatus.getExecutingAndFinalStatuses())); - } - // if project owner, show all tickets of the project - } else { - // find tickets related to all projects that the current user joins in - Map> currentUserProjectId2ResourceRoleNames = - resourceRoleService.getProjectId2ResourceRoleNames(); - Set ownerProjectIds = currentUserProjectId2ResourceRoleNames.entrySet().stream() - .filter(entry -> entry.getValue().contains(ResourceRoleName.OWNER)) - .map(Entry::getKey) - .collect(Collectors.toSet()); - Set otherRoleProjectIds = new HashSet<>(currentUserProjectId2ResourceRoleNames.keySet()); - otherRoleProjectIds.removeAll(ownerProjectIds); - - - Specification ownerSpecification = - Specification.where(FlowInstanceViewSpecs.projectIdIn(ownerProjectIds)); - - Specification otherRoleSpecification = - Specification.where(FlowInstanceViewSpecs.projectIdIn(otherRoleProjectIds)) - .and(FlowInstanceViewSpecs.leftJoinFlowInstanceApprovalView( - resourceRoleIdentifiers, authenticationFacade.currentUserId(), - FlowNodeStatus.getExecutingAndFinalStatuses())); - - if (CollectionUtils.isEmpty(ownerProjectIds)) { - specification = specification.and(otherRoleSpecification); - } else if (CollectionUtils.isEmpty(otherRoleProjectIds)) { - specification = specification.and(ownerSpecification); - } else { - specification = specification.and(ownerSpecification.or(otherRoleSpecification)); - } + if (CollectionUtils.isNotEmpty(params.getProjectIds())) { + specification = specification.and(FlowInstanceViewSpecs.projectIdIn(params.getProjectIds())); + } else { + Set joinedProjectIds = userService.getCurrentUserJoinedProjectIds(); + // if the user does not join any projects in team space, then return empty directly + if (CollectionUtils.isEmpty(joinedProjectIds) + && authenticationFacade.currentOrganization().getType() == OrganizationType.TEAM) { + return Page.empty(); } + specification = + specification.and(FlowInstanceViewSpecs.projectIdIn(userService.getCurrentUserJoinedProjectIds())); + } + if (params.getContainsAll()) { return flowInstanceViewRepository.findAll(specification, pageable).map(FlowInstanceEntity::from); } - if (!params.getApproveByCurrentUser() && params.getCreatedByCurrentUser()) { + if (params.getCreatedByCurrentUser()) { // created by current user - specification = specification.and(FlowInstanceViewSpecs.projectIdEquals(params.getProjectId())) - .and(FlowInstanceViewSpecs.creatorIdEquals(authenticationFacade.currentUserId())); - return flowInstanceViewRepository.findAll(specification, pageable).map(FlowInstanceEntity::from); - } else if (params.getApproveByCurrentUser() && !params.getCreatedByCurrentUser()) { + specification = + specification.and(FlowInstanceViewSpecs.creatorIdEquals(authenticationFacade.currentUserId())); + } + if (params.getApproveByCurrentUser()) { + Set resourceRoleIdentifiers = userService.getCurrentUserResourceRoleIdentifiers(); + // does not join any project, so there does not exist any tickets to approve if (CollectionUtils.isEmpty(resourceRoleIdentifiers)) { return Page.empty(); } - // approving by current user - specification = - specification.and(FlowInstanceViewSpecs.projectIdEquals(params.getProjectId())) - .and(FlowInstanceViewSpecs.leftJoinFlowInstanceApprovalView( - resourceRoleIdentifiers, null, FlowNodeStatus.getExecutingStatuses())); - return flowInstanceViewRepository.findAll(specification, pageable).map(FlowInstanceEntity::from); - } else { - throw new UnsupportedOperationException("Unsupported list flow instance query"); + specification = specification.and(FlowInstanceViewSpecs.leftJoinFlowInstanceApprovalView( + resourceRoleIdentifiers, null, FlowNodeStatus.getExecutingStatuses())); } + return flowInstanceViewRepository.findAll(specification, pageable).map(FlowInstanceEntity::from); } public List listByIds(@NonNull Collection ids) { @@ -564,24 +526,19 @@ public List listByIds(@NonNull Collection ids) { } public FlowInstanceDetailResp detail(@NotNull Long id) { - return mapFlowInstance(id, flowInstance -> { + return mapFlowInstanceWithReadPermission(id, flowInstance -> { FlowInstanceMapper instanceMapper = mapperFactory.generateMapperByInstance(flowInstance, false); FlowNodeInstanceMapper nodeInstanceMapper = mapperFactory.generateNodeMapperByInstance(flowInstance, false); return instanceMapper.map(flowInstance, nodeInstanceMapper); - }, false); + }); } @Transactional(rollbackFor = Exception.class) public FlowInstanceDetailResp cancel(@NotNull Long id, Boolean skipAuth) { - FlowInstance flowInstance = mapFlowInstance(id, flowInst -> flowInst, skipAuth); + FlowInstance flowInstance = mapFlowInstanceWithWritePermission(id, flowInst -> flowInst); return cancel(flowInstance, skipAuth); } - public FlowInstanceDetailResp cancelNotCheckPermission(@NotNull Long id) { - FlowInstance flowInstance = mapFlowInstance(id, flowInst -> flowInst, false); - return cancel(flowInstance, false); - } - public Map getStatus(Set ids) { Specification specification = Specification.where(FlowInstanceSpecs.idIn(ids)) .and(FlowInstanceSpecs.organizationIdEquals(authenticationFacade.currentOrganizationId())); @@ -737,30 +694,37 @@ public FlowInstanceDetailResp reject(@NotNull Long id, return FlowInstanceDetailResp.withIdAndType(id, getTaskByFlowInstanceId(id).getTaskType()); } - public T mapFlowInstance(@NonNull Long flowInstanceId, Function function, Boolean skipAuth) { + public T mapFlowInstance(@NonNull Long flowInstanceId, Function mapper, + Consumer checkAuth) { Optional optional = flowFactory.getFlowInstance(flowInstanceId); FlowInstance flowInstance = optional.orElseThrow(() -> new NotFoundException(ResourceType.ODC_FLOW_INSTANCE, "id", flowInstanceId)); try { - if (!skipAuth) { - boolean isProjectOwner = flowInstance.getProjectId() != null && projectPermissionValidator - .hasProjectRole(flowInstance.getProjectId(), Collections.singletonList(ResourceRoleName.OWNER)); - if (!Objects.equals(authenticationFacade.currentUserId(), flowInstance.getCreatorId()) - && !isProjectOwner) { - List entities = approvalPermissionService.getApprovableApprovalInstances(); - Set flowInstanceIds = entities.stream().map(UserTaskInstanceEntity::getFlowInstanceId) - .collect(Collectors.toSet()); - PreConditions.validExists(ResourceType.ODC_FLOW_INSTANCE, "id", flowInstanceId, - () -> flowInstanceIds.contains(flowInstanceId)); - } - permissionValidator.checkCurrentOrganization(flowInstance); + if (checkAuth != null) { + checkAuth.accept(flowInstance); } - return function.apply(flowInstance); + return mapper.apply(flowInstance); } finally { flowInstance.dealloc(); } } + public T mapFlowInstanceWithReadPermission(@NonNull Long flowInstanceId, Function mapper) { + return mapFlowInstance(flowInstanceId, mapper, flowPermissionHelper.withProjectMemberCheck()); + } + + public T mapFlowInstanceWithWritePermission(@NonNull Long flowInstanceId, Function mapper) { + return mapFlowInstance(flowInstanceId, mapper, flowPermissionHelper.withProjectOwnerCheck()); + } + + public T mapFlowInstanceWithApprovalPermission(@NonNull Long flowInstanceId, Function mapper) { + return mapFlowInstance(flowInstanceId, mapper, flowPermissionHelper.withApprovableCheck()); + } + + public T mapFlowInstanceWithoutPermissionCheck(@NonNull Long flowInstanceId, Function mapper) { + return mapFlowInstance(flowInstanceId, mapper, flowPermissionHelper.skipCheck()); + } + public TaskEntity getTaskByFlowInstanceId(Long id) { List entities = serviceTaskRepository .findAll(ServiceTaskInstanceSpecs.flowInstanceIdEquals(id)) @@ -1095,24 +1059,22 @@ private FlowInstanceConfigurer buildConfigurer( private void completeApprovalInstance(@NonNull Long flowInstanceId, @NonNull Consumer consumer, Boolean skipAuth) { List instances = - mapFlowInstance(flowInstanceId, flowInstance -> flowInstance.filterInstanceNode(instance -> { - if (instance.getNodeType() != FlowNodeType.APPROVAL_TASK) { - return false; - } - return instance.getStatus() == FlowNodeStatus.EXECUTING - || instance.getStatus() == FlowNodeStatus.WAIT_FOR_CONFIRM; - }).stream().map(instance -> { - Verify.verify(instance instanceof FlowApprovalInstance, "FlowApprovalInstance's type is illegal"); - return (FlowApprovalInstance) instance; - }).collect(Collectors.toList()), skipAuth); + mapFlowInstanceWithApprovalPermission(flowInstanceId, + flowInstance -> flowInstance.filterInstanceNode(instance -> { + if (instance.getNodeType() != FlowNodeType.APPROVAL_TASK) { + return false; + } + return instance.getStatus() == FlowNodeStatus.EXECUTING + || instance.getStatus() == FlowNodeStatus.WAIT_FOR_CONFIRM; + }).stream().map(instance -> { + Verify.verify(instance instanceof FlowApprovalInstance, + "FlowApprovalInstance's type is illegal"); + return (FlowApprovalInstance) instance; + }).collect(Collectors.toList())); PreConditions.validExists(ResourceType.ODC_FLOW_APPROVAL_INSTANCE, "flowInstanceId", flowInstanceId, () -> instances.size() > 0); Verify.singleton(instances, "ApprovalInstance"); FlowApprovalInstance target = instances.get(0); - if (!skipAuth) { - PreConditions.validExists(ResourceType.ODC_FLOW_INSTANCE, "id", flowInstanceId, - () -> approvalPermissionService.isApprovable(target.getId())); - } Verify.verify(target.isPresentOnThisMachine(), "Approval instance is not on this machine"); consumer.accept(target); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowPermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowPermissionHelper.java new file mode 100644 index 0000000000..f11eaa53ce --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowPermissionHelper.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.flow; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.exception.AccessDeniedException; +import com.oceanbase.odc.metadb.flow.UserTaskInstanceEntity; +import com.oceanbase.odc.service.flow.instance.FlowInstance; +import com.oceanbase.odc.service.iam.HorizontalDataPermissionValidator; +import com.oceanbase.odc.service.iam.ProjectPermissionValidator; +import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; + +/** + * @Author: Lebie + * @Date: 2024/10/25 15:29 + * @Description: [] + */ +@Component +public class FlowPermissionHelper { + @Autowired + private ProjectPermissionValidator projectPermissionValidator; + + @Autowired + private ApprovalPermissionService approvalPermissionService; + + @Autowired + private AuthenticationFacade authenticationFacade; + + @Autowired + private HorizontalDataPermissionValidator horizontalDataPermissionValidator; + + + public Consumer withProjectMemberCheck() { + return withProjectPermissionCheck( + flowInstance -> flowInstance.getProjectId() != null && projectPermissionValidator + .hasProjectRole(flowInstance.getProjectId(), ResourceRoleName.all())); + } + + public Consumer withProjectOwnerCheck() { + return withProjectPermissionCheck( + flowInstance -> flowInstance.getProjectId() != null && projectPermissionValidator + .hasProjectRole(flowInstance.getProjectId(), + Collections.singletonList(ResourceRoleName.OWNER))); + } + + public Consumer withApprovableCheck() { + return flowInstance -> { + List entities = approvalPermissionService.getApprovableApprovalInstances(); + Set flowInstanceIds = entities.stream().map(UserTaskInstanceEntity::getFlowInstanceId) + .collect(Collectors.toSet()); + PreConditions.validExists(ResourceType.ODC_FLOW_INSTANCE, "id", flowInstance.getId(), + () -> flowInstanceIds.contains(flowInstance.getId())); + horizontalDataPermissionValidator.checkCurrentOrganization(flowInstance); + }; + } + + public Consumer withCreatorCheck() { + return flowInstance -> { + if (!Objects.equals(authenticationFacade.currentUserId(), flowInstance.getCreatorId())) { + throw new AccessDeniedException(); + } + horizontalDataPermissionValidator.checkCurrentOrganization(flowInstance); + }; + } + + public Consumer skipCheck() { + return flowInstance -> { + }; + } + + private Consumer withProjectPermissionCheck(Predicate predicate) { + return flowInstance -> { + if (!Objects.equals(authenticationFacade.currentUserId(), flowInstance.getCreatorId()) + && !predicate.test(flowInstance)) { + throw new AccessDeniedException(); + } + horizontalDataPermissionValidator.checkCurrentOrganization(flowInstance); + }; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceLoggerService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceLoggerService.java index 8dc2eed412..f5e73bf95a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceLoggerService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceLoggerService.java @@ -68,11 +68,15 @@ public class FlowTaskInstanceLoggerService { @Autowired private ScheduleLogProperties loggerProperty; + @Autowired + private FlowPermissionHelper flowPermissionHelper; + @SneakyThrows public String getLogContent(OdcTaskLogLevel level, Long flowInstanceId) { try { Optional taskEntityOptional = - flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, false); + flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, + flowPermissionHelper.withProjectMemberCheck()); return getLogContent(taskEntityOptional, level, flowInstanceId); } catch (Exception e) { log.warn("Task log file not found, flowInstanceId={}", flowInstanceId, e); @@ -84,7 +88,7 @@ public String getLogContent(OdcTaskLogLevel level, Long flowInstanceId) { public String getLogContentWithoutPermission(OdcTaskLogLevel level, Long flowInstanceId) { try { Optional taskEntityOptional = - flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, true); + flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, null); return getLogContent(taskEntityOptional, level, flowInstanceId); } catch (Exception e) { log.warn("get log failed, task log file not found, flowInstanceId={}", flowInstanceId); @@ -129,7 +133,8 @@ public String getLogContent(Optional taskEntityOptional, OdcTaskLogL @SneakyThrows private InputStream downloadLog(Long flowInstanceId) { Optional taskEntityOptional = - flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, false); + flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, + flowPermissionHelper.withProjectMemberCheck()); TaskEntity taskEntity = taskEntityOptional .orElseThrow(() -> new NotFoundException(ErrorCodes.NotFound, new Object[] {flowInstanceId}, ErrorCodes.TaskLogNotFound.getLocalizedMessage(new Object[] {"Id", flowInstanceId}))); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java index 5a1642c3f0..b4b69d7ff4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java @@ -30,6 +30,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -77,6 +78,7 @@ import com.oceanbase.odc.service.dispatch.DispatchResponse; import com.oceanbase.odc.service.dispatch.RequestDispatcher; import com.oceanbase.odc.service.dispatch.TaskDispatchChecker; +import com.oceanbase.odc.service.flow.instance.FlowInstance; import com.oceanbase.odc.service.flow.instance.FlowTaskInstance; import com.oceanbase.odc.service.flow.model.BinaryDataResult; import com.oceanbase.odc.service.flow.model.ByteArrayDataResult; @@ -160,6 +162,8 @@ public class FlowTaskInstanceService { private TaskFrameworkEnabledProperties taskFrameworkProperties; @Autowired private FlowTaskInstanceLoggerService flowTaskInstanceLoggerService; + @Autowired + private FlowPermissionHelper flowPermissionHelper; @Value("${odc.task.async.result-preview-max-size-bytes:5242880}") private long resultPreviewMaxSizeBytes; @@ -169,7 +173,8 @@ public class FlowTaskInstanceService { @Transactional(rollbackFor = Exception.class) public FlowInstanceDetailResp executeTask(@NotNull Long id) throws IOException { List instances = - filterTaskInstance(id, instance -> instance.getStatus() == FlowNodeStatus.PENDING, false); + filterTaskInstance(id, instance -> instance.getStatus() == FlowNodeStatus.PENDING, + flowPermissionHelper.withCreatorCheck()); PreConditions.validExists(ResourceType.ODC_FLOW_TASK_INSTANCE, "flowInstanceId", id, () -> instances.size() > 0); Verify.singleton(instances, "FlowTaskInstance"); @@ -205,18 +210,12 @@ public InputStreamResource downloadLog(@NotNull Long flowInstanceId) { return flowTaskInstanceLoggerService.downloadLogFile(flowInstanceId); } - public List getResult(@NotNull Long id, boolean skipAuth) throws IOException { - TaskEntity task = flowInstanceService.getTaskByFlowInstanceId(id); - if (task.getTaskType() == TaskType.ONLINE_SCHEMA_CHANGE || task.getTaskType() == TaskType.EXPORT - || task.getTaskType() == TaskType.MULTIPLE_ASYNC) { - return getTaskResultFromEntity(task, true); - } - Optional taskEntityOptional = getCompleteTaskEntity(id, skipAuth); - if (!taskEntityOptional.isPresent()) { - return Collections.emptyList(); - } - TaskEntity taskEntity = taskEntityOptional.get(); - return getTaskResultFromEntity(taskEntity, true); + public List getResult(@NotNull Long id) throws IOException { + return getResult(id, flowPermissionHelper.withProjectMemberCheck()); + } + + public List getResultSkipPermissionCheck(@NotNull Long id) throws IOException { + return getResult(id, flowPermissionHelper.skipCheck()); } public List getTaskResultFromEntity(@NotNull TaskEntity taskEntity, @@ -261,11 +260,10 @@ public List getTaskResultFromEntity(@NotNull TaskEntit } public List getResult( - @NotNull Long flowInstanceId, @NotNull Long nodeInstanceId, boolean skipAuth) throws IOException { - List taskInstances = this.flowInstanceService.mapFlowInstance( + @NotNull Long flowInstanceId, @NotNull Long nodeInstanceId) throws IOException { + List taskInstances = this.flowInstanceService.mapFlowInstanceWithReadPermission( flowInstanceId, i -> i.filterInstanceNode(f -> f instanceof FlowTaskInstance) - .stream().map(f -> (FlowTaskInstance) f).collect(Collectors.toList()), - skipAuth); + .stream().map(f -> (FlowTaskInstance) f).collect(Collectors.toList())); Optional target = taskInstances.stream() .filter(f -> f.getId().equals(nodeInstanceId)).findFirst(); if (!target.isPresent()) { @@ -340,7 +338,8 @@ public List download(@NonNull Long flowInstanceId, String targ public List downRollbackPlanResult(@NonNull Long flowInstanceId) throws IOException { Optional taskEntityOptional = getTaskEntity(flowInstanceId, - instance -> instance.getStatus().isFinalStatus() && instance.getTaskType() == TaskType.ASYNC, false); + instance -> instance.getStatus().isFinalStatus() && instance.getTaskType() == TaskType.ASYNC, + flowPermissionHelper.withProjectMemberCheck()); PreConditions.validExists(ResourceType.ODC_FILE, "flowInstanceId", flowInstanceId, taskEntityOptional::isPresent); TaskEntity taskEntity = taskEntityOptional.get(); @@ -536,7 +535,8 @@ public List getAsyncDownloadUrl(Long id, List objectIds, String } public List getExecuteResult(Long flowInstanceId) throws IOException { - Optional taskEntityOptional = getCompleteTaskEntity(flowInstanceId, false); + Optional taskEntityOptional = getCompleteTaskEntity(flowInstanceId, + flowPermissionHelper.withProjectMemberCheck()); if (!taskEntityOptional.isPresent()) { return Collections.emptyList(); } @@ -565,6 +565,22 @@ public List getExecuteResult(Long flowInstanceId) throws IOExc } } + + private List getResult(@NotNull Long id, Consumer checkAuth) + throws IOException { + TaskEntity task = flowInstanceService.getTaskByFlowInstanceId(id); + if (task.getTaskType() == TaskType.ONLINE_SCHEMA_CHANGE || task.getTaskType() == TaskType.EXPORT + || task.getTaskType() == TaskType.MULTIPLE_ASYNC) { + return getTaskResultFromEntity(task, true); + } + Optional taskEntityOptional = getCompleteTaskEntity(id, checkAuth); + if (!taskEntityOptional.isPresent()) { + return Collections.emptyList(); + } + TaskEntity taskEntity = taskEntityOptional.get(); + return getTaskResultFromEntity(taskEntity, true); + } + private Set getDownloadImportFileNames(@NonNull TaskEntity taskEntity, String targetFileName) { DataTransferConfig config = JsonUtils.fromJson( taskEntity.getParametersJson(), DataTransferConfig.class); @@ -585,7 +601,7 @@ private Set getDownloadImportFileNames(@NonNull TaskEntity taskEntity, S } private List filterTaskInstance(@NonNull Long flowInstanceId, - @NonNull Predicate predicate, boolean skipAuth) { + @NonNull Predicate predicate, Consumer checkAuth) { return flowInstanceService.mapFlowInstance(flowInstanceId, flowInstance -> flowInstance.filterInstanceNode(instance -> { if (instance.getNodeType() != FlowNodeType.SERVICE_TASK) { @@ -595,7 +611,7 @@ private List filterTaskInstance(@NonNull Long flowInstanceId, }).stream().map(instance -> { Verify.verify(instance instanceof FlowTaskInstance, "FlowTaskInstance's type is illegal"); return (FlowTaskInstance) instance; - }).collect(Collectors.toList()), skipAuth); + }).collect(Collectors.toList()), checkAuth); } private List getMultipleAsyncResult(@NonNull TaskEntity taskEntity) { @@ -702,11 +718,11 @@ private List innerGetResult(@NonNull TaskEntity ta return Collections.singletonList(detail); } - private Optional getCompleteTaskEntity(@NonNull Long flowInstanceId, boolean skipAuth) { + private Optional getCompleteTaskEntity(@NonNull Long flowInstanceId, Consumer checkAuth) { return getTaskEntity(flowInstanceId, i -> i.getStatus().isFinalStatus() && i.getTaskType() != TaskType.SQL_CHECK && i.getTaskType() != TaskType.PRE_CHECK - && i.getTaskType() != TaskType.GENERATE_ROLLBACK, skipAuth); + && i.getTaskType() != TaskType.GENERATE_ROLLBACK, checkAuth); } private Optional getDownloadableTaskEntity(@NonNull Long flowInstanceId) { @@ -725,20 +741,21 @@ private Optional getDownloadableTaskEntity(@NonNull Long flowInstanc && instance.getTaskType() != TaskType.APPLY_DATABASE_PERMISSION && instance.getTaskType() != TaskType.APPLY_TABLE_PERMISSION; } - }, false); + }, flowPermissionHelper.withProjectMemberCheck()); } - public Optional getLogDownloadableTaskEntity(@NotNull Long flowInstanceId, boolean skipAuth) { + public Optional getLogDownloadableTaskEntity(@NotNull Long flowInstanceId, + Consumer checkAuth) { return getTaskEntity(flowInstanceId, instance -> (instance.getStatus().isFinalStatus() || instance.getStatus() == FlowNodeStatus.EXECUTING) && instance.getTaskType() != TaskType.SQL_CHECK && instance.getTaskType() != TaskType.PRE_CHECK && instance.getTaskType() != TaskType.GENERATE_ROLLBACK, - skipAuth); + checkAuth); } private Optional getTaskEntity(@NonNull Long flowInstanceId, - @NonNull Predicate predicate, boolean skipAuth) { - List taskInstances = filterTaskInstance(flowInstanceId, predicate, skipAuth); + @NonNull Predicate predicate, Consumer checkAuth) { + List taskInstances = filterTaskInstance(flowInstanceId, predicate, checkAuth); if (CollectionUtils.isEmpty(taskInstances)) { return Optional.empty(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java index aac09f1535..18cbee0a9a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -41,11 +40,7 @@ import com.google.common.collect.Sets; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.StringUtils; -import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.constant.TaskType; -import com.oceanbase.odc.core.shared.exception.NotFoundException; -import com.oceanbase.odc.metadb.collaboration.ProjectEntity; -import com.oceanbase.odc.metadb.collaboration.ProjectRepository; import com.oceanbase.odc.metadb.connection.ConnectionConfigRepository; import com.oceanbase.odc.metadb.connection.ConnectionEntity; import com.oceanbase.odc.metadb.connection.ConnectionSpecs; @@ -73,6 +68,8 @@ import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.metadb.task.TaskRepository; import com.oceanbase.odc.metadb.task.TaskSpecs; +import com.oceanbase.odc.service.collaboration.project.ProjectService; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; @@ -93,6 +90,7 @@ import com.oceanbase.odc.service.integration.model.IntegrationConfig; import com.oceanbase.odc.service.integration.model.TemplateVariables; import com.oceanbase.odc.service.integration.model.TemplateVariables.Variable; +import com.oceanbase.odc.service.permission.project.ApplyProjectParameter; import com.oceanbase.odc.service.regulation.risklevel.RiskLevelMapper; import lombok.NonNull; @@ -142,7 +140,8 @@ public class FlowResponseMapperFactory { @Autowired private AuthenticationFacade authenticationFacade; @Autowired - private ProjectRepository projectRepository; + private ProjectService projectService; + private final ConnectionMapper connectionMapper = ConnectionMapper.INSTANCE; private final RiskLevelMapper riskLevelMapper = RiskLevelMapper.INSTANCE; @@ -328,43 +327,21 @@ private FlowInstanceMapper generateMapper(@NonNull Collection flowInstance Set databaseIds = taskId2TaskEntity.values().stream() .map(TaskEntity::getDatabaseId) .filter(Objects::nonNull).collect(Collectors.toSet()); - Set sourceDatabaseIdsInComparisonTask = new HashSet<>(); - Set targetDatabaseIdsInComparisonTask = new HashSet<>(); - taskId2TaskEntity.values().stream() - .filter(task -> task.getTaskType().equals(TaskType.STRUCTURE_COMPARISON)) - .forEach(task -> { - DBStructureComparisonParameter parameter = JsonUtils.fromJson( - task.getParametersJson(), DBStructureComparisonParameter.class); - sourceDatabaseIdsInComparisonTask.add(parameter.getSourceDatabaseId()); - targetDatabaseIdsInComparisonTask.add(parameter.getTargetDatabaseId()); - }); - databaseIds.addAll(targetDatabaseIdsInComparisonTask); + databaseIds.addAll(collectDBStructureComparisonDatabaseIds(taskId2TaskEntity)); + Set projectIds = new HashSet<>(); + Map id2Project = new HashMap<>(); if (CollectionUtils.isNotEmpty(databaseIds)) { id2Database = databaseService.listDatabasesByIds(databaseIds).stream() .collect(Collectors.toMap(Database::getId, database -> database)); - - // set project name for structure comparison task - Set projectIds = sourceDatabaseIdsInComparisonTask.stream() - .map(id2Database::get) - .filter(Objects::nonNull) - .map(database -> database.getProject().getId()) - .collect(Collectors.toSet()); - Map id2ProjectEntity = projectRepository.findByIdIn(projectIds).stream() - .collect(Collectors.toMap(ProjectEntity::getId, Function.identity())); - - for (Long id : sourceDatabaseIdsInComparisonTask) { - Database database = id2Database.get(id); - if (Objects.nonNull(database) && Objects.nonNull(database.getProject())) { - Long projectId = database.getProject().getId(); - if (Objects.nonNull(projectId)) { - ProjectEntity projectEntity = Optional.ofNullable(id2ProjectEntity.get(projectId)).orElseThrow( - () -> new NotFoundException(ResourceType.ODC_PROJECT, "projectId", projectId)); - database.getProject().setName(projectEntity.getName()); - } - } - } + projectIds.addAll(id2Database.values().stream().map(db -> db.getProject().getId()) + .filter(Objects::nonNull).collect(Collectors.toSet())); + } + projectIds.addAll(collectApplyProjectIds(taskId2TaskEntity)); + if (CollectionUtils.isNotEmpty(projectIds)) { + id2Project = projectService.listByIds(projectIds).stream() + .collect(Collectors.toMap(Project::getId, project -> project, (a, b) -> a)); } /** * find the ConnectionConfig associated with each Database @@ -417,7 +394,9 @@ private FlowInstanceMapper generateMapper(@NonNull Collection flowInstance .getRiskLevelByRiskLevelId( id -> riskLevelRepository.findById(id).map(riskLevelMapper::entityToModel).orElse(null)) .getCandidatesByFlowInstanceId(candidatesByFlowInstanceIds::get) - .getDatabaseById(id2Database::get).build(); + .getDatabaseById(id2Database::get) + .getProjectById(id2Project::get) + .build(); } public Map> getUserId2Roles(@NonNull Collection userIds, boolean skipAuth) { @@ -473,4 +452,31 @@ private List listConnectionsByConnectionIdsWithoutPermissionCh return connectionRepository.findAll(specification); } + private Set collectDBStructureComparisonDatabaseIds(Map taskId2TaskEntity) { + Set targetDatabaseIdsInComparisonTask = new HashSet<>(); + taskId2TaskEntity.values().stream() + .filter(task -> task.getTaskType().equals(TaskType.STRUCTURE_COMPARISON)) + .forEach(task -> { + DBStructureComparisonParameter parameter = JsonUtils.fromJson( + task.getParametersJson(), DBStructureComparisonParameter.class); + targetDatabaseIdsInComparisonTask.add(parameter.getTargetDatabaseId()); + }); + return targetDatabaseIdsInComparisonTask; + } + + private Set collectApplyProjectIds(Map taskId2TaskEntity) { + Set applyProjectIds = taskId2TaskEntity.values().stream() + .filter(task -> task.getTaskType() == TaskType.APPLY_PROJECT_PERMISSION) + .map(task -> { + ApplyProjectParameter parameter = + JsonUtils.fromJson(task.getParametersJson(), ApplyProjectParameter.class); + if (Objects.nonNull(parameter) && Objects.nonNull(parameter.getProject())) { + return parameter.getProject().getId(); + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + return applyProjectIds; + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java index 04497a9690..8b194eee11 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java @@ -41,6 +41,7 @@ import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.plugin.task.api.datatransfer.model.DataTransferConfig; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.model.InnerUser; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.flow.instance.BaseFlowNodeInstance; @@ -99,6 +100,7 @@ public class FlowInstanceDetailResp { private boolean rollbackable; private Date completeTime; private List nodeList; + private Project project; @Getter @Builder @@ -113,6 +115,7 @@ public static class FlowInstanceMapper { private final Function> getExecutionStrategyByFlowInstanceId; private final Function getRiskLevelByRiskLevelId; private final Function> getCandidatesByFlowInstanceId; + private final Function getProjectById; public FlowInstanceDetailResp map(@NonNull FlowInstanceEntity entity) { FlowInstanceDetailResp resp = FlowInstanceDetailResp.withId(entity.getId()); @@ -120,6 +123,7 @@ public FlowInstanceDetailResp map(@NonNull FlowInstanceEntity entity) { resp.setCreateTime(entity.getCreateTime()); resp.setStatus(entity.getStatus()); resp.setProjectId(entity.getProjectId()); + resp.setProject(getProjectById.apply(entity.getProjectId())); resp.setOrganizationId(entity.getOrganizationId()); Set candidates = getCandidatesByFlowInstanceId.apply(entity.getId()); if (candidates != null) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/QueryFlowInstanceParams.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/QueryFlowInstanceParams.java index 3d02e2714f..e8f309f09e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/QueryFlowInstanceParams.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/QueryFlowInstanceParams.java @@ -17,6 +17,7 @@ import java.util.Date; import java.util.List; +import java.util.Set; import com.oceanbase.odc.core.shared.constant.FlowStatus; import com.oceanbase.odc.core.shared.constant.TaskType; @@ -49,5 +50,5 @@ public class QueryFlowInstanceParams { private Boolean containsAll; private Long parentInstanceId; - private Long projectId; + private Set projectIds; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java index 1a97020597..c886cc8488 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java @@ -118,15 +118,17 @@ public Set getResourceRoleIdentifiersByUserId(long organizationId, long @SkipAuthorize public Map> getProjectId2ResourceRoleNames() { - return getProjectId2ResourceRoleNames(authenticationFacade.currentUserId()); + return getProjectId2ResourceRoleNames(authenticationFacade.currentUserId(), + authenticationFacade.currentOrganizationId()); } @SkipAuthorize - public Map> getProjectId2ResourceRoleNames(Long userId) { + public Map> getProjectId2ResourceRoleNames(Long userId, Long organizationId) { Map id2ResourceRoles = resourceRoleRepository.findByResourceType(ResourceType.ODC_PROJECT) .stream().map(resourceRoleMapper::entityToModel) .collect(Collectors.toMap(ResourceRole::getId, resourceRole -> resourceRole, (v1, v2) -> v2)); - return userResourceRoleRepository.findByUserIdAndResourceType(userId, ResourceType.ODC_PROJECT).stream() + return userResourceRoleRepository + .findByUserIdAndResourceTypeAndOrganizationId(userId, ResourceType.ODC_PROJECT, organizationId).stream() .collect(Collectors.groupingBy(UserResourceRoleEntity::getResourceId, Collectors.mapping( e -> id2ResourceRoles.get(e.getResourceRoleId()).getRoleName(), Collectors.toSet()))); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java index 8d83c9e701..33e84a0220 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java @@ -472,6 +472,10 @@ public Set getCurrentUserResourceRoleIdentifiers() { return resourceRoleService.getResourceRoleIdentifiersByUserId(currentOrganizationId, currentUserId); } + public Set getCurrentUserJoinedProjectIds() { + return resourceRoleService.getProjectId2ResourceRoleNames().keySet(); + } + private void acquirePermissions(@NonNull Collection users) { List managementPermissions = new ArrayList<>(); List operationPermissions = new ArrayList<>(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 2db30b196e..c2953d31c6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -45,6 +45,7 @@ import org.springframework.integration.jdbc.lock.JdbcLockRegistry; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.StringUtils; @@ -738,13 +739,15 @@ public Page list(@NotNull Pageable pageable, @NotNull Quer params.getCreator()).stream().map(User::getId).collect(Collectors.toSet())); } if (authenticationFacade.currentOrganization().getType() == OrganizationType.TEAM) { - Set projectIds = params.getProjectId() == null - ? projectService.getMemberProjectIds(authenticationFacade.currentUserId()) - : Collections.singleton(params.getProjectId()); - if (projectIds.isEmpty()) { + Set joinedProjectIds = projectService.getMemberProjectIds(authenticationFacade.currentUserId()); + if (CollectionUtils.isEmpty(joinedProjectIds)) { return Page.empty(); } - params.setProjectIds(projectIds); + if (CollectionUtils.isEmpty(params.getProjectIds())) { + params.setProjectIds(joinedProjectIds); + } else { + params.getProjectIds().retainAll(joinedProjectIds); + } } params.setOrganizationId(authenticationFacade.currentOrganizationId()); Page returnValue = scheduleRepository.find(pageable, params); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java index dbb5b66cb5..cb2c6e0331 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java @@ -44,6 +44,8 @@ import com.oceanbase.odc.metadb.schedule.ScheduleEntity; import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; +import com.oceanbase.odc.service.collaboration.project.ProjectService; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.model.InnerUser; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.database.DatabaseService; @@ -102,6 +104,8 @@ public class ScheduleResponseMapperFactory { private FlowInstanceRepository flowInstanceRepository; @Autowired private ApprovalPermissionService approvalPermissionService; + @Autowired + private ProjectService projectService; @@ -264,6 +268,10 @@ public Map generateHistoryScheduleList(@NonNull Coll .filter(entry -> flowInstanceId2Candidates.get(entry.getValue()) != null).collect( Collectors.toMap(Entry::getKey, entry -> flowInstanceId2Candidates.get(entry.getValue()))); + Map id2Project = projectService + .listByIds(schedules.stream().map(ScheduleEntity::getProjectId).collect(Collectors.toSet())).stream() + .collect(Collectors.toMap(Project::getId, o -> o, (o1, o2) -> o2)); + return schedules.stream().map(schedule -> { ScheduleOverviewHist resp = new ScheduleOverviewHist(); resp.setId(schedule.getId()); @@ -284,6 +292,7 @@ public Map generateHistoryScheduleList(@NonNull Coll if (CollectionUtils.isNotEmpty(candidates)) { resp.setCandidateApprovers(candidates.stream().map(InnerUser::new).collect(Collectors.toSet())); } + resp.setProject(id2Project.get(schedule.getProjectId())); return resp; }).collect(Collectors.toMap(ScheduleOverviewHist::getId, o -> o)); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleOverviewHist.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleOverviewHist.java index 6951c4f09a..aa6d373412 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleOverviewHist.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleOverviewHist.java @@ -19,6 +19,7 @@ import java.util.Set; import com.oceanbase.odc.common.i18n.Internationalizable; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.model.InnerUser; import lombok.Data; @@ -51,4 +52,6 @@ public class ScheduleOverviewHist { @Internationalizable private String description; + private Project project; + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/shadowtable/ShadowTableComparingService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/shadowtable/ShadowTableComparingService.java index 97d261ba67..3e90026066 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/shadowtable/ShadowTableComparingService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/shadowtable/ShadowTableComparingService.java @@ -222,6 +222,7 @@ private void checkPermission(ShadowTableComparingTaskEntity taskEntity) { } return; } - flowInstanceService.mapFlowInstance(taskEntity.getFlowInstanceId(), flowInstance -> flowInstance, false); + flowInstanceService.mapFlowInstanceWithReadPermission(taskEntity.getFlowInstanceId(), + flowInstance -> flowInstance); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java index 2d9376e480..092505140b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java @@ -147,6 +147,7 @@ public List batchCreateTaskResults(List flowInstance, false); + flowInstanceService.mapFlowInstanceWithReadPermission(taskEntity.getFlowInstanceId(), + flowInstance -> flowInstance); } } From f8cb35bf9a69efd9671c044a8db334b0a64ef857 Mon Sep 17 00:00:00 2001 From: yizhou Date: Fri, 8 Nov 2024 11:55:06 +0800 Subject: [PATCH 024/118] feat(web): session cookie secure default false (#3798) --- server/odc-server/src/main/resources/data.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-server/src/main/resources/data.sql b/server/odc-server/src/main/resources/data.sql index 3d43646391..c2f78f4bc9 100644 --- a/server/odc-server/src/main/resources/data.sql +++ b/server/odc-server/src/main/resources/data.sql @@ -851,4 +851,4 @@ INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('o -- v4.3.3 -- INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('server.servlet.session.cookie.secure', - 'true', 'Enable secure cookie or not, default value true') ON DUPLICATE KEY UPDATE `id`=`id`; + 'false', 'Enable secure cookie or not, default value false') ON DUPLICATE KEY UPDATE `id`=`id`; From df9685b90d1c641d6330e6d3583f4b5b21675dfe Mon Sep 17 00:00:00 2001 From: LioRoger Date: Tue, 12 Nov 2024 16:45:23 +0800 Subject: [PATCH 025/118] feat(task): refactor task interface, introduce task runtime component (#3807) * refactor task interface, introduce task runtime component * change agent.runtime package class visiable scope to default * change agent.runtime package class visiable scope to default * change agent.runtime package class visiable scope to default * fix TaskApplicationTest --- .../odc/service/task/TaskApplicationTest.java | 121 ------------- server/odc-server/pom.xml | 5 + .../com/oceanbase/odc/agent/OdcAgent.java | 1 + .../runtime}/DefaultTaskResultBuilder.java | 10 +- .../odc/agent/runtime/EmbedServer.java | 2 +- .../agent/runtime/ExecutorRequestHandler.java | 9 +- .../odc/agent/runtime/ExitHelper.java | 2 +- .../agent/{ => runtime}/TaskApplication.java | 6 +- .../odc/agent/runtime/TaskContainer.java} | 142 +++++++-------- .../odc/agent/runtime/TaskExecutor.java | 4 +- .../odc/agent/runtime/TaskFactory.java | 7 +- .../odc/agent/runtime/TaskMonitor.java | 53 ++---- .../odc/agent/runtime}/TaskReporter.java | 4 +- .../odc/agent/runtime/TaskRuntimeInfo.java | 8 +- .../agent/runtime/ThreadPoolTaskExecutor.java | 72 ++------ .../odc/agent/runtime}/EmbedServerTest.java | 5 +- .../odc/agent/runtime/SimpleTask.java | 62 +++++++ .../agent/runtime/TaskApplicationTest.java | 99 ++++++++++ .../odc/agent/runtime/TaskContainerTest.java | 105 +++++++++++ .../odc/agent/runtime/TaskMonitorTest.java | 57 +----- .../com/oceanbase/odc/service/task/Task.java | 29 +-- .../odc/service/task/TaskContext.java | 7 - .../odc/service/task/base/TaskBase.java | 88 +++++++++ .../base/dataarchive/DataArchiveTask.java | 35 ++-- .../databasechange/DatabaseChangeTask.java | 20 +- .../LogicalDatabaseChangeTask.java | 29 ++- .../task/base/precheck/PreCheckTask.java | 21 ++- .../task/base/rollback/RollbackPlanTask.java | 21 ++- .../task/base/sqlplan/SqlPlanTask.java | 21 ++- .../odc/service/task/dummy/DummyTask.java | 13 +- .../task/executor/task/BaseTaskTest.java | 171 ------------------ 31 files changed, 598 insertions(+), 631 deletions(-) delete mode 100644 server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/executor => odc-server/src/main/java/com/oceanbase/odc/agent/runtime}/DefaultTaskResultBuilder.java (80%) rename server/odc-server/src/main/java/com/oceanbase/odc/agent/{ => runtime}/TaskApplication.java (96%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java => odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskContainer.java} (52%) rename server/{odc-service/src/main/java/com/oceanbase/odc/service/task/executor => odc-server/src/main/java/com/oceanbase/odc/agent/runtime}/TaskReporter.java (96%) rename server/{integration-test/src/test/java/com/oceanbase/odc/service/task => odc-server/src/test/java/com/oceanbase/odc/agent/runtime}/EmbedServerTest.java (92%) create mode 100644 server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/SimpleTask.java create mode 100644 server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskApplicationTest.java create mode 100644 server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/TaskBase.java delete mode 100644 server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java deleted file mode 100644 index 915d0c390a..0000000000 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.task; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.commons.collections4.CollectionUtils; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import com.oceanbase.odc.TestConnectionUtil; -import com.oceanbase.odc.agent.TaskApplication; -import com.oceanbase.odc.agent.runtime.TaskRuntimeInfo; -import com.oceanbase.odc.agent.runtime.ThreadPoolTaskExecutor; -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.core.shared.PreConditions; -import com.oceanbase.odc.core.shared.constant.ConnectType; -import com.oceanbase.odc.core.shared.constant.ErrorCodes; -import com.oceanbase.odc.core.shared.constant.TaskErrorStrategy; -import com.oceanbase.odc.core.shared.constant.TaskType; -import com.oceanbase.odc.service.cloud.model.CloudProvider; -import com.oceanbase.odc.service.connection.model.ConnectionConfig; -import com.oceanbase.odc.service.flow.task.model.DatabaseChangeParameters; -import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; -import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTask; -import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.caller.JobEnvironmentFactory; -import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; -import com.oceanbase.odc.service.task.enums.JobStatus; -import com.oceanbase.odc.service.task.enums.TaskRunMode; -import com.oceanbase.odc.service.task.schedule.DefaultJobContextBuilder; -import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; -import com.oceanbase.odc.service.task.schedule.JobDefinition; -import com.oceanbase.odc.service.task.schedule.JobIdentity; -import com.oceanbase.odc.service.task.util.JobUtils; - -/** - * @author yaobin - * @date 2023-12-14 - * @since 4.2.4 - */ -@Ignore("manual run this case") -public class TaskApplicationTest extends BaseJobTest { - - @Test - public void test_executeDatabaseChangeTask_run() { - Long exceptedTaskId = System.currentTimeMillis(); - JobIdentity jobIdentity = JobIdentity.of(exceptedTaskId); - - setJobContextInSystemProperty(jobIdentity); - startTaskApplication(); - assertRunningResult(jobIdentity); - } - - - private void setJobContextInSystemProperty(JobIdentity jobIdentity) { - JobDefinition jd = buildJobDefinition(); - JobContext jc = new DefaultJobContextBuilder().build(jobIdentity, jd); - Map envMap = new JobEnvironmentFactory().build(jc, TaskRunMode.PROCESS); - JobUtils.encryptEnvironments(envMap); - envMap.forEach(System::setProperty); - } - - private void assertRunningResult(JobIdentity ji) { - - try { - Thread.sleep(60 * 1000L); - TaskRuntimeInfo taskRuntimeInfo = ThreadPoolTaskExecutor.getInstance().getTaskRuntimeInfo(ji); - Task task = taskRuntimeInfo.getTask(); - Assert.assertSame(JobStatus.DONE, task.getStatus()); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - } - - - private void startTaskApplication() { - new Thread(() -> new TaskApplication().run(null)).start(); - } - - private JobDefinition buildJobDefinition() { - Long exceptedTaskId = System.currentTimeMillis(); - DatabaseChangeParameters parameters = new DatabaseChangeParameters(); - parameters.setSqlContent(String.format("CREATE TABLE %s (id int(10))", "t_" + exceptedTaskId)); - parameters.setErrorStrategy(TaskErrorStrategy.ABORT.name()); - PreConditions.validArgumentState( - parameters.getSqlContent() != null || CollectionUtils.isNotEmpty(parameters.getSqlObjectIds()), - ErrorCodes.BadArgument, new Object[] {"sql"}, "input sql is empty"); - - ConnectionConfig config = TestConnectionUtil.getTestConnectionConfig(ConnectType.OB_MYSQL); - Map jobData = new HashMap<>(); - jobData.put(JobParametersKeyConstants.META_TASK_PARAMETER_JSON, JsonUtils.toJson(parameters)); - jobData.put(JobParametersKeyConstants.CONNECTION_CONFIG, JobUtils.toJson(config)); - jobData.put(JobParametersKeyConstants.FLOW_INSTANCE_ID, exceptedTaskId + ""); - jobData.put(JobParametersKeyConstants.CURRENT_SCHEMA, config.getDefaultSchema()); - jobData.put(JobParametersKeyConstants.TASK_EXECUTION_TIMEOUT_MILLIS, 30 * 60 * 1000 + ""); - ObjectStorageConfiguration storageConfig = new ObjectStorageConfiguration(); - storageConfig.setCloudProvider(CloudProvider.NONE); - - return DefaultJobDefinition.builder().jobClass(DatabaseChangeTask.class) - .jobType(TaskType.ASYNC.name()) - .jobParameters(jobData) - .build(); - } -} diff --git a/server/odc-server/pom.xml b/server/odc-server/pom.xml index 188ee7f13d..6566807cc8 100644 --- a/server/odc-server/pom.xml +++ b/server/odc-server/pom.xml @@ -148,6 +148,11 @@ mockito-core test + + org.mockito + mockito-inline + test + diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/OdcAgent.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/OdcAgent.java index 2003795c22..b7bd7150e9 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/OdcAgent.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/OdcAgent.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.agent; +import com.oceanbase.odc.agent.runtime.TaskApplication; import com.oceanbase.odc.server.module.Modules; import lombok.extern.slf4j.Slf4j; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResultBuilder.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java similarity index 80% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResultBuilder.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java index e70a0256be..6be6feadf5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResultBuilder.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java @@ -13,10 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor; +package com.oceanbase.odc.agent.runtime; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.service.task.Task; +import com.oceanbase.odc.service.task.executor.DefaultTaskResult; import com.oceanbase.odc.service.task.util.JobUtils; /** @@ -24,12 +25,13 @@ * @date 2024-01-12 * @since 4.2.4 */ -public class DefaultTaskResultBuilder { +class DefaultTaskResultBuilder { - public static DefaultTaskResult build(Task task) { + public static DefaultTaskResult build(TaskContainer taskContainer) { DefaultTaskResult result = new DefaultTaskResult(); + Task task = taskContainer.getTask(); result.setResultJson(JsonUtils.toJson(task.getTaskResult())); - result.setStatus(task.getStatus()); + result.setStatus(taskContainer.getStatus()); result.setProgress(task.getProgress()); result.setJobIdentity(task.getJobContext().getJobIdentity()); result.setExecutorEndpoint(JobUtils.getExecutorPoint()); diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/EmbedServer.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/EmbedServer.java index 0fe7916461..f753a7b12c 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/EmbedServer.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/EmbedServer.java @@ -65,7 +65,7 @@ * @since 4.2.4 */ @Slf4j -public class EmbedServer { +class EmbedServer { private ExecutorRequestHandler requestHandler; private Thread thread; diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java index f7674798b0..2079aa349e 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java @@ -27,7 +27,6 @@ import com.oceanbase.odc.service.common.util.UrlUtils; import com.oceanbase.odc.service.task.constants.JobExecutorUrls; import com.oceanbase.odc.service.task.executor.DefaultTaskResult; -import com.oceanbase.odc.service.task.executor.DefaultTaskResultBuilder; import com.oceanbase.odc.service.task.executor.logger.LogBiz; import com.oceanbase.odc.service.task.executor.logger.LogBizImpl; import com.oceanbase.odc.service.task.executor.logger.LogUtils; @@ -43,7 +42,7 @@ * @since 4.2.4 */ @Slf4j -public class ExecutorRequestHandler { +class ExecutorRequestHandler { private final Pattern queryLogUrlPattern = Pattern.compile(String.format(JobExecutorUrls.QUERY_LOG, "([0-9]+)")); private final Pattern stopTaskPattern = Pattern.compile(String.format(JobExecutorUrls.STOP_TASK, "([0-9]+)")); @@ -87,7 +86,7 @@ public SuccessResponse process(HttpMethod httpMethod, String uri, String if (matcher.find()) { JobIdentity ji = getJobIdentity(matcher); TaskRuntimeInfo runtimeInfo = ThreadPoolTaskExecutor.getInstance().getTaskRuntimeInfo(ji); - boolean result = runtimeInfo.getTask().modify(JobUtils.fromJsonToMap(requestData)); + boolean result = runtimeInfo.getTaskContainer().modify(JobUtils.fromJsonToMap(requestData)); return Responses.ok(result); } @@ -96,11 +95,11 @@ public SuccessResponse process(HttpMethod httpMethod, String uri, String JobIdentity ji = getJobIdentity(matcher); TaskRuntimeInfo runtimeInfo = ThreadPoolTaskExecutor.getInstance().getTaskRuntimeInfo(ji); TaskMonitor taskMonitor = runtimeInfo.getTaskMonitor(); - DefaultTaskResult result = DefaultTaskResultBuilder.build(runtimeInfo.getTask()); + DefaultTaskResult result = DefaultTaskResultBuilder.build(runtimeInfo.getTaskContainer()); if (taskMonitor != null && MapUtils.isNotEmpty(taskMonitor.getLogMetadata())) { result.setLogMetadata(taskMonitor.getLogMetadata()); // assign final error message - DefaultTaskResultBuilder.assignErrorMessage(result, taskMonitor.getError()); + DefaultTaskResultBuilder.assignErrorMessage(result, runtimeInfo.getTaskContainer().getError()); taskMonitor.markLogMetaCollected(); log.info("Task log metadata collected, ji={}.", ji.getId()); } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExitHelper.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExitHelper.java index a84d8c51b3..472039836e 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExitHelper.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExitHelper.java @@ -25,7 +25,7 @@ * @since 4.2.4 */ @Slf4j -public class ExitHelper { +class ExitHelper { private static final CountDownLatch LATCH = new CountDownLatch(1); diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/TaskApplication.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskApplication.java similarity index 96% rename from server/odc-server/src/main/java/com/oceanbase/odc/agent/TaskApplication.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskApplication.java index 13b83693dc..c3b3e8274f 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/TaskApplication.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskApplication.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.agent; +package com.oceanbase.odc.agent.runtime; import java.io.File; import java.net.URI; @@ -24,10 +24,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LoggerContext; -import com.oceanbase.odc.agent.runtime.EmbedServer; -import com.oceanbase.odc.agent.runtime.ExitHelper; -import com.oceanbase.odc.agent.runtime.TaskFactory; -import com.oceanbase.odc.agent.runtime.ThreadPoolTaskExecutor; import com.oceanbase.odc.common.trace.TaskContextHolder; import com.oceanbase.odc.common.trace.TraceContextHolder; import com.oceanbase.odc.common.util.StringUtils; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskContainer.java similarity index 52% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskContainer.java index a63a753181..c780b20144 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/BaseTask.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskContainer.java @@ -13,21 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.base; +package com.oceanbase.odc.agent.runtime; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; +import com.oceanbase.odc.service.task.ExceptionListener; import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.caller.DefaultJobContext; import com.oceanbase.odc.service.task.caller.JobContext; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; /** @@ -35,26 +35,61 @@ * @date 2023/11/22 20:16 */ @Slf4j -public abstract class BaseTask implements Task { - +final class TaskContainer implements ExceptionListener { + // if task has been closed private final AtomicBoolean closed = new AtomicBoolean(false); - protected TaskContext context; - private DefaultJobContext jobContext; + // status maintained by task container private volatile TaskStatus status = TaskStatus.PREPARING; + // task event listener for task event notify + @Getter + protected final TaskMonitor taskMonitor; + // task context for task to init + protected final TaskContext taskContext; + // task instance holding + @Getter + private final Task task; + // only save latest exception if any + // it will be cleaned if been fetched + protected AtomicReference latestException = new AtomicReference<>(); + + public TaskContainer(JobContext jobContext, CloudObjectStorageService cloudObjectStorageService, + TaskReporter taskReporter, // assignable for test + Task task) { + this.taskContext = createTaskContext(jobContext, cloudObjectStorageService); + this.task = task; + this.taskMonitor = new TaskMonitor(this, taskReporter, cloudObjectStorageService); + } + + protected TaskContext createTaskContext(JobContext jobContext, + CloudObjectStorageService cloudObjectStorageService) { + ExceptionListener exceptionListener = this; + return new TaskContext() { + @Override + public ExceptionListener getExceptionListener() { + return exceptionListener; + } + @Override + public JobContext getJobContext() { + return jobContext; + } - @Override - public void start(TaskContext taskContext) { - this.context = taskContext; - jobContext = copyJobContext(taskContext); - log.info("Start task, id={}.", jobContext.getJobIdentity().getId()); - log.info("Init task parameters success, id={}.", jobContext.getJobIdentity().getId()); + @Override + public CloudObjectStorageService getSharedStorage() { + return cloudObjectStorageService; + } + }; + } + /** + * 1. init task 2. run task 3. maintain status + */ + public void runTask() { try { - doInit(jobContext); + task.init(taskContext); updateStatus(TaskStatus.RUNNING); - context.getTaskEventListener().onTaskStart(this); - if (doStart(jobContext, taskContext)) { + taskMonitor.monitor(); + if (task.start()) { updateStatus(TaskStatus.DONE); } else { updateStatus(TaskStatus.FAILED); @@ -62,19 +97,18 @@ public void start(TaskContext taskContext) { } catch (Throwable e) { log.warn("Task failed, id={}.", getJobId(), e); updateStatus(TaskStatus.FAILED); - taskContext.getExceptionListener().onException(e); + onException(e); } finally { close(); } } - @Override public boolean stop() { try { if (getStatus().isTerminated()) { log.warn("Task is already finished and cannot be canceled, id={}, status={}.", getJobId(), getStatus()); } else { - doStop(); + task.stop(); // doRefresh cannot execute if update status to 'canceled'. updateStatus(TaskStatus.CANCELED); } @@ -87,7 +121,6 @@ public boolean stop() { } } - @Override public boolean modify(Map jobParameters) { if (Objects.isNull(jobParameters) || jobParameters.isEmpty()) { log.warn("Job parameter cannot be null, id={}", getJobId()); @@ -97,12 +130,7 @@ public boolean modify(Map jobParameters) { log.warn("Task is already finished, cannot modify parameters, id={}", getJobId()); return false; } - jobContext.setJobParameters(Collections.unmodifiableMap(jobParameters)); - try { - afterModifiedJobParameters(); - } catch (Exception e) { - log.warn("Do after modified job parameters failed", e); - } + task.modify(jobParameters); return true; } @@ -110,75 +138,41 @@ public boolean modify(Map jobParameters) { private void close() { if (closed.compareAndSet(false, true)) { try { - doClose(); + task.close(); } catch (Throwable e) { // do nothing } log.info("Task completed, id={}, status={}.", getJobId(), getStatus()); - if (null != context) { - context.getTaskEventListener().onTaskFinalize(this); - } + taskMonitor.finalWork(); } } - @Override public TaskStatus getStatus() { return status; } - @Override - public JobContext getJobContext() { - return jobContext; - } - protected void updateStatus(TaskStatus status) { + private void updateStatus(TaskStatus status) { log.info("Update task status, id={}, status={}.", getJobId(), status); this.status = status; } protected Map getJobParameters() { - return jobContext.getJobParameters(); + return task.getJobContext().getJobParameters(); } private Long getJobId() { - return getJobContext().getJobIdentity().getId(); + return task.getJobContext().getJobIdentity().getId(); } - protected abstract void doInit(JobContext context) throws Exception; - - /** - * start a task return succeed or failed after completed. - * - * @return return true if execute succeed, else return false - */ - protected abstract boolean doStart(JobContext context, TaskContext taskContext) throws Exception; - - protected abstract void doStop() throws Exception; - - /** - * task can release relational resource in this method - */ - protected abstract void doClose() throws Exception; - - protected void afterModifiedJobParameters() throws Exception { - // do nothing + public Throwable getError() { + Throwable e = latestException.getAndSet(null); + log.info("retrieve exception = {}", null == e ? null : e.getMessage()); + return e; } - // deep copy job context - protected DefaultJobContext copyJobContext(TaskContext taskContext) { - DefaultJobContext ret = new DefaultJobContext(); - JobContext src = taskContext.getJobContext(); - ret.setJobIdentity(src.getJobIdentity()); - if (null != src.getJobProperties()) { - ret.setJobProperties(new HashMap<>(src.getJobProperties())); - } - if (null != src.getJobParameters()) { - ret.setJobParameters(Collections.unmodifiableMap(new HashMap<>(src.getJobParameters()))); - } - ret.setJobClass(src.getJobClass()); - if (null != src.getHostUrls()) { - ret.setHostUrls(new ArrayList<>(src.getHostUrls())); - } - return ret; + public void onException(Throwable e) { + log.info("found exception", e); + this.latestException.set(e); } } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java index 94f1f353fa..346d74052d 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java @@ -23,7 +23,7 @@ * @author gaoda.xy * @date 2023/11/24 11:18 */ -public interface TaskExecutor { +interface TaskExecutor { void execute(Task task, JobContext jc); @@ -31,4 +31,6 @@ public interface TaskExecutor { TaskRuntimeInfo getTaskRuntimeInfo(JobIdentity ji); + boolean taskExist(JobIdentity ji); + } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskFactory.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskFactory.java index 4bb8fbc1a8..7ee7addd1b 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskFactory.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskFactory.java @@ -16,22 +16,21 @@ package com.oceanbase.odc.agent.runtime; import com.oceanbase.odc.service.task.Task; -import com.oceanbase.odc.service.task.base.BaseTask; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; /** * @author gaoda.xy * @date 2023/11/24 11:01 */ -public class TaskFactory { +class TaskFactory { - public static BaseTask create(String jobClass) { + public static Task create(String jobClass) { try { Class c = Class.forName(jobClass); if (!Task.class.isAssignableFrom(c)) { throw new TaskRuntimeException("Job class is not implements Task. name={}" + jobClass); } - return (BaseTask) c.newInstance(); + return (Task) c.newInstance(); } catch (Exception e) { throw new TaskRuntimeException(e); } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java index 0507da329c..b8ec027d39 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java @@ -23,25 +23,20 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.lang3.StringUtils; import com.google.common.annotations.VisibleForTesting; -import com.oceanbase.odc.common.util.ObjectUtil; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.core.task.TaskThreadFactory; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; -import com.oceanbase.odc.service.task.ExceptionListener; import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.constants.JobAttributeKeyConstants; import com.oceanbase.odc.service.task.constants.JobConstants; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.constants.JobServerUrls; import com.oceanbase.odc.service.task.executor.DefaultTaskResult; -import com.oceanbase.odc.service.task.executor.DefaultTaskResultBuilder; import com.oceanbase.odc.service.task.executor.HeartbeatRequest; -import com.oceanbase.odc.service.task.executor.TaskReporter; import com.oceanbase.odc.service.task.executor.TraceDecoratorThreadFactory; import com.oceanbase.odc.service.task.executor.logger.LogBizImpl; import com.oceanbase.odc.service.task.util.JobUtils; @@ -54,25 +49,22 @@ * @since 4.2.4 */ @Slf4j -public class TaskMonitor implements ExceptionListener { +class TaskMonitor { private static final int REPORT_RESULT_RETRY_TIMES = Integer.MAX_VALUE; private static final long WAIT_AFTER_LOG_METADATA_COLLECT_MILLS = 5000L; private final TaskReporter reporter; - private final Task task; + private final TaskContainer taskContainer; private final CloudObjectStorageService cloudObjectStorageService; private volatile long startTimeMilliSeconds; private ScheduledExecutorService reportScheduledExecutor; private ScheduledExecutorService heartScheduledExecutor; private Map logMetadata = new HashMap<>(); private AtomicLong logMetaCollectedMillis = new AtomicLong(0L); - // only save latest exception if any - // it will be cleaned if been fetched - protected AtomicReference latestException = new AtomicReference<>(); - public TaskMonitor(Task task, TaskReporter taskReporter, + public TaskMonitor(TaskContainer task, TaskReporter taskReporter, CloudObjectStorageService cloudObjectStorageService) { - this.task = task; + this.taskContainer = task; this.reporter = taskReporter; this.cloudObjectStorageService = cloudObjectStorageService; } @@ -94,9 +86,9 @@ public void monitor() { new TraceDecoratorThreadFactory(new TaskThreadFactory(("Task-Monitor-Job-" + getJobId()))); this.reportScheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); reportScheduledExecutor.scheduleAtFixedRate(() -> { - if (isTimeout() && !getTask().getStatus().isTerminated()) { + if (isTimeout() && !getTaskContainer().getStatus().isTerminated()) { log.info("Task timeout, try stop, jobId={}", getJobId()); - getTask().stop(); + getTaskContainer().stop(); } try { if (JobUtils.getExecutorPort().isPresent()) { @@ -151,17 +143,16 @@ protected void reportTaskResult() { if (JobUtils.isReportDisabled()) { return; } - DefaultTaskResult taskResult = DefaultTaskResultBuilder.build(getTask()); - DefaultTaskResult copiedResult = ObjectUtil.deepCopy(taskResult, DefaultTaskResult.class); - if (copiedResult.getStatus().isTerminated()) { + DefaultTaskResult taskResult = DefaultTaskResultBuilder.build(getTaskContainer()); + if (taskResult.getStatus().isTerminated()) { log.info("job {} status {} is terminate, monitor report be ignored.", - copiedResult.getJobIdentity().getId(), copiedResult.getStatus()); + taskResult.getJobIdentity().getId(), taskResult.getStatus()); return; } - getReporter().report(JobServerUrls.TASK_UPLOAD_RESULT, copiedResult); + getReporter().report(JobServerUrls.TASK_UPLOAD_RESULT, taskResult); log.info("Report task info, id: {}, status: {}, progress: {}%, result: {}", getJobId(), - copiedResult.getStatus(), String.format("%.2f", copiedResult.getProgress()), getTask().getTaskResult()); + taskResult.getStatus(), String.format("%.2f", taskResult.getProgress()), getTask().getTaskResult()); } @VisibleForTesting @@ -184,7 +175,7 @@ protected boolean isTimeout() { @VisibleForTesting protected void doFinal() { - DefaultTaskResult finalResult = DefaultTaskResultBuilder.build(getTask()); + DefaultTaskResult finalResult = DefaultTaskResultBuilder.build(getTaskContainer()); // Report final result log.info("Task id: {}, finished with status: {}, start to report final result", getJobId(), finalResult.getStatus()); @@ -203,7 +194,7 @@ protected void doFinal() { if (JobUtils.isReportEnabled()) { // assign error for last report - DefaultTaskResultBuilder.assignErrorMessage(finalResult, getError()); + DefaultTaskResultBuilder.assignErrorMessage(finalResult, taskContainer.getError()); // Report finish signal to task server reportTaskResultWithRetry(finalResult, REPORT_RESULT_RETRY_TIMES, JobConstants.REPORT_TASK_INFO_INTERVAL_SECONDS); @@ -281,11 +272,12 @@ private HeartbeatRequest buildHeartRequest() { return request; } - private Task getTask() { - return task; + private TaskContainer getTaskContainer() { + return taskContainer; } - private TaskReporter getReporter() { + @VisibleForTesting + protected TaskReporter getReporter() { return reporter; } @@ -293,14 +285,7 @@ private Long getJobId() { return getTask().getJobContext().getJobIdentity().getId(); } - public Throwable getError() { - Throwable e = latestException.getAndSet(null); - log.info("retrieve exception = {}", null == e ? null : e.getMessage()); - return e; - } - - public void onException(Throwable e) { - log.info("found exception", e); - this.latestException.set(e); + private Task getTask() { + return getTaskContainer().getTask(); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskReporter.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskReporter.java similarity index 96% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskReporter.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskReporter.java index 448cd52e07..8ea5fb4c86 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskReporter.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskReporter.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor; +package com.oceanbase.odc.agent.runtime; import java.text.MessageFormat; import java.util.List; @@ -32,7 +32,7 @@ * @date 2023/11/30 19:41 */ @Slf4j -public class TaskReporter { +class TaskReporter { private final List hostUrls; diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskRuntimeInfo.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskRuntimeInfo.java index 56483ee6cf..b596132db2 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskRuntimeInfo.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskRuntimeInfo.java @@ -17,8 +17,6 @@ import java.util.concurrent.Future; -import com.oceanbase.odc.service.task.Task; - import lombok.Data; /** @@ -26,8 +24,8 @@ * @date 2024/10/24 14:32 */ @Data -public class TaskRuntimeInfo { - private final Task task; - private final Future future; +class TaskRuntimeInfo { + private final TaskContainer taskContainer; + private Future future; private final TaskMonitor taskMonitor; } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java index f6c24f8a2d..e20d3bffc9 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java @@ -30,12 +30,8 @@ import com.oceanbase.odc.core.task.TaskThreadFactory; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; -import com.oceanbase.odc.service.task.ExceptionListener; import com.oceanbase.odc.service.task.Task; -import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.TaskEventListener; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.executor.TaskReporter; import com.oceanbase.odc.service.task.executor.TraceDecoratorThreadFactory; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.util.CloudObjectStorageServiceBuilder; @@ -50,7 +46,7 @@ * @date 2023/11/24 11:22 */ @Slf4j -public class ThreadPoolTaskExecutor implements TaskExecutor { +class ThreadPoolTaskExecutor implements TaskExecutor { private static final TaskExecutor TASK_EXECUTOR = new ThreadPoolTaskExecutor(); private final Map tasks = new ConcurrentHashMap<>(); @@ -75,63 +71,20 @@ synchronized public void execute(Task task, JobContext jc) { } // init cloud objet storage service and task monitor CloudObjectStorageService cloudObjectStorageService = buildCloudStorageService(jc); - TaskMonitor taskMonitor = new TaskMonitor(task, new TaskReporter(jc.getHostUrls()), cloudObjectStorageService); - + TaskReporter taskReporter = new TaskReporter(jc.getHostUrls()); + TaskContainer taskContainer = new TaskContainer<>(jc, cloudObjectStorageService, taskReporter, task); + TaskRuntimeInfo taskRuntimeInfo = new TaskRuntimeInfo(taskContainer, taskContainer.getTaskMonitor()); + // first put data in map, avoid concurrent thread access caused task not found exception + tasks.put(jobIdentity, taskRuntimeInfo); Future future = executor.submit(() -> { try { - task.start(new TaskContext() { - @Override - public ExceptionListener getExceptionListener() { - return taskMonitor; - } - - @Override - public JobContext getJobContext() { - return jc; - } - - @Override - public TaskEventListener getTaskEventListener() { - return taskEventListener(taskMonitor); - } - - @Override - public CloudObjectStorageService getSharedStorage() { - return cloudObjectStorageService; - } - }); + taskContainer.runTask(); } catch (Exception e) { log.error("Task start failed, jobIdentity={}.", jobIdentity.getId(), e); - taskMonitor.onException(e); + taskContainer.onException(e); } }); - tasks.put(jobIdentity, new TaskRuntimeInfo(task, future, taskMonitor)); - } - - /** - * build task event listener - * - * @param taskMonitor - * @return - */ - private TaskEventListener taskEventListener(TaskMonitor taskMonitor) { - return new TaskEventListener() { - @Override - public void onTaskStart(Task task) { - taskMonitor.monitor(); - } - - @Override - public void onTaskStop(Task task) {} - - @Override - public void onTaskModify(Task task) {} - - @Override - public void onTaskFinalize(Task task) { - taskMonitor.finalWork(); - } - }; + taskRuntimeInfo.setFuture(future); } /** @@ -156,7 +109,7 @@ protected CloudObjectStorageService buildCloudStorageService(JobContext jobConte @Override public boolean cancel(JobIdentity ji) { TaskRuntimeInfo runtimeInfo = getTaskRuntimeInfo(ji); - Task task = runtimeInfo.getTask(); + TaskContainer task = runtimeInfo.getTaskContainer(); Future stopFuture = executor.submit(task::stop); boolean result = false; try { @@ -186,4 +139,9 @@ public TaskRuntimeInfo getTaskRuntimeInfo(JobIdentity ji) { PreConditions.notNull(runtimeInfo, "task", "Task not found, jobIdentity=" + ji.getId()); return runtimeInfo; } + + @Override + public boolean taskExist(JobIdentity ji) { + return tasks.get(ji) != null; + } } diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/EmbedServerTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/EmbedServerTest.java similarity index 92% rename from server/integration-test/src/test/java/com/oceanbase/odc/service/task/EmbedServerTest.java rename to server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/EmbedServerTest.java index ebf5553d9f..b253f53eb6 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/EmbedServerTest.java +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/EmbedServerTest.java @@ -13,14 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.oceanbase.odc.service.task; +package com.oceanbase.odc.agent.runtime; import org.junit.Ignore; import org.junit.Test; -import com.oceanbase.odc.agent.runtime.EmbedServer; - import lombok.extern.slf4j.Slf4j; /** diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/SimpleTask.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/SimpleTask.java new file mode 100644 index 0000000000..6315299dde --- /dev/null +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/SimpleTask.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.agent.runtime; + +import com.oceanbase.odc.service.task.base.TaskBase; +import com.oceanbase.odc.service.task.caller.JobContext; + +/** + * @author longpeng.zlp + * @date 2024/11/12 15:07 + */ +public final class SimpleTask extends TaskBase { + private final boolean shouldThrowException; + + public SimpleTask() { + this.shouldThrowException = false; + } + + public SimpleTask(boolean shouldThrowException) { + this.shouldThrowException = shouldThrowException; + } + + @Override + protected void doInit(JobContext context) throws Exception {} + + @Override + public boolean start() throws Exception { + if (shouldThrowException) { + throw new IllegalStateException("exception should be thrown"); + } + return true; + } + + @Override + public void stop() throws Exception {} + + @Override + public void close() throws Exception {} + + @Override + public double getProgress() { + return 100; + } + + @Override + public String getTaskResult() { + return "res"; + } +} diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskApplicationTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskApplicationTest.java new file mode 100644 index 0000000000..6c8880075e --- /dev/null +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskApplicationTest.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.agent.runtime; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; + +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.task.caller.DefaultJobContext; +import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; +import com.oceanbase.odc.service.task.schedule.JobIdentity; +import com.oceanbase.odc.service.task.util.JobUtils; + +/** + * @author yaobin + * @date 2023-12-14 + * @since 4.2.4 + */ +public class TaskApplicationTest { + @Test + public void test_executeDatabaseChangeTask_run() { + Long exceptedTaskId = System.currentTimeMillis(); + JobIdentity jobIdentity = JobIdentity.of(exceptedTaskId); + setJobContextInSystemProperty(jobIdentity); + startTaskApplication(); + assertRunningResult(jobIdentity); + } + + + private void setJobContextInSystemProperty(JobIdentity jobIdentity) { + Map envMap = buildConfig(jobIdentity); + JobUtils.encryptEnvironments(envMap); + envMap.forEach(System::setProperty); + } + + private void assertRunningResult(JobIdentity ji) { + + try { + long endTime = System.currentTimeMillis() + 20000; + TaskExecutor taskExecutor = ThreadPoolTaskExecutor.getInstance(); + while (System.currentTimeMillis() < endTime) { + if (!taskExecutor.taskExist(ji)) { + Thread.sleep(1000); + continue; + } + TaskRuntimeInfo taskRuntimeInfo = taskExecutor.getTaskRuntimeInfo(ji); + TaskContainer task = taskRuntimeInfo.getTaskContainer(); + task.taskMonitor.markLogMetaCollected(); + if (TaskStatus.DONE == task.getStatus()) { + return; + } else { + Thread.sleep(1000); + } + } + // not check pass + Assert.assertFalse(true); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + } + + private Map buildConfig(JobIdentity jobIdentity) { + Map ret = new HashMap<>(); + ret.put(JobEnvKeyConstants.ODC_TASK_RUN_MODE, "K8S"); + ret.put(JobEnvKeyConstants.ODC_BOOT_MODE, "TASK_EXECUTOR"); + ret.put(JobEnvKeyConstants.ODC_EXECUTOR_USER_ID, "1"); + DefaultJobContext defaultJobContext = new DefaultJobContext(); + defaultJobContext.setJobClass(SimpleTask.class.getName()); + defaultJobContext.setJobIdentity(jobIdentity); + ret.put(JobEnvKeyConstants.ODC_JOB_CONTEXT, JsonUtils.toJson(defaultJobContext)); + ret.put(JobEnvKeyConstants.ODC_LOG_DIRECTORY, "log"); + ret.put(JobEnvKeyConstants.ODC_EXECUTOR_PORT, "8080"); + ret.put(JobEnvKeyConstants.REPORT_ENABLED, "false"); + return ret; + } + + private void startTaskApplication() { + new Thread(() -> new TaskApplication().run(null)).start(); + } + +} diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java new file mode 100644 index 0000000000..059608adc2 --- /dev/null +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.agent.runtime; + +import java.util.HashMap; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import com.oceanbase.odc.common.util.SystemUtils; +import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; +import com.oceanbase.odc.service.task.Task; +import com.oceanbase.odc.service.task.caller.DefaultJobContext; +import com.oceanbase.odc.service.task.caller.JobContext; +import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; +import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.schedule.JobIdentity; + +/** + * @author longpeng.zlp + * @date 2024/10/11 14:11 + */ +public class TaskContainerTest { + private DefaultJobContext jobContext; + + @Before + public void init() { + jobContext = new DefaultJobContext(); + jobContext.setJobParameters(new HashMap<>()); + jobContext.setJobProperties(new HashMap<>()); + JobIdentity jobIdentity = new JobIdentity(); + jobIdentity.setId(1L); + jobContext.setJobIdentity(jobIdentity); + jobContext.setJobClass(SimpleTask.class.getName()); + } + + + @Test + public void testExceptionListenerNormal() { + try (MockedStatic mockSystemUtil = Mockito.mockStatic(SystemUtils.class)) { + mockSystemUtil.when(() -> { + SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_EXECUTOR_PORT); + }).thenReturn("9099"); + SimpleTask dummyBaseTask = new SimpleTask(false); + TaskContainer taskContainer = buildTaskContainer(jobContext, dummyBaseTask); + taskContainer.runTask(); + TaskReporter taskReporter = taskContainer.taskMonitor.getReporter(); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DefaultTaskResult.class); + Mockito.verify(taskReporter).report(ArgumentMatchers.any(), argumentCaptor.capture()); + Assert.assertNull(argumentCaptor.getValue().getErrorMessage()); + } + } + + @Test + public void testExceptionListenerWithException() { + try (MockedStatic mockSystemUtil = Mockito.mockStatic(SystemUtils.class)) { + mockSystemUtil.when(() -> { + SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_EXECUTOR_PORT); + }).thenReturn("9099"); + SimpleTask dummyBaseTask = new SimpleTask(true); + TaskContainer taskContainer = buildTaskContainer(jobContext, dummyBaseTask); + taskContainer.runTask(); + TaskReporter taskReporter = taskContainer.taskMonitor.getReporter(); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DefaultTaskResult.class); + Mockito.verify(taskReporter).report(ArgumentMatchers.any(), argumentCaptor.capture()); + Assert.assertEquals(argumentCaptor.getValue().getErrorMessage(), "exception should be thrown"); + } + } + + @Test + public void testTaskContainerOnException() { + TaskContainer taskContainer = new TaskContainer<>(jobContext, Mockito.mock( + CloudObjectStorageService.class), Mockito.mock(TaskReporter.class), new SimpleTask(false)); + Assert.assertNull(taskContainer.getError()); + taskContainer.onException(new Throwable("error")); + Throwable ex = taskContainer.getError(); + Assert.assertEquals(ex.getMessage(), "error"); + Assert.assertNull(taskContainer.getError()); + } + + private static TaskContainer buildTaskContainer(JobContext jobContext, Task task) { + TaskReporter taskReporter = Mockito.mock(TaskReporter.class); + Mockito.when(taskReporter.report(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(true); + return new TaskContainer<>(jobContext, Mockito.mock(CloudObjectStorageService.class), taskReporter, task); + } +} diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java index 3e52e15c4b..c1fc08ca38 100644 --- a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java @@ -15,42 +15,25 @@ */ package com.oceanbase.odc.agent.runtime; -import java.util.Map; - import org.junit.Assert; import org.junit.Test; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; -import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; -import com.oceanbase.odc.service.task.Task; -import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.executor.DefaultTaskResult; -import com.oceanbase.odc.service.task.executor.TaskReporter; /** * @author longpeng.zlp * @date 2024/11/7 17:45 */ public class TaskMonitorTest { - @Test - public void testTaskMonitorOnException() { - TaskMonitor taskMonitor = new TaskMonitor(new MockBaseTask(), Mockito.mock(TaskReporter.class), Mockito.mock( - CloudObjectStorageService.class)); - Assert.assertNull(taskMonitor.getError()); - taskMonitor.onException(new Throwable("error")); - Throwable ex = taskMonitor.getError(); - Assert.assertEquals(ex.getMessage(), "error"); - Assert.assertNull(taskMonitor.getError()); - } @Test public void testTaskMonitorReportRetryFailed() { TaskReporter taskReporter = Mockito.mock(TaskReporter.class); Mockito.when(taskReporter.report(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); - TaskMonitor taskMonitor = new TaskMonitor(new MockBaseTask(), taskReporter, Mockito.mock( + TaskMonitor taskMonitor = new TaskMonitor(Mockito.mock(TaskContainer.class), taskReporter, Mockito.mock( CloudObjectStorageService.class)); Assert.assertFalse(taskMonitor.reportTaskResultWithRetry(new DefaultTaskResult(), 3, 1)); } @@ -59,44 +42,8 @@ public void testTaskMonitorReportRetryFailed() { public void testTaskMonitorReportRetrySuccess() { TaskReporter taskReporter = Mockito.mock(TaskReporter.class); Mockito.when(taskReporter.report(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(true); - TaskMonitor taskMonitor = new TaskMonitor(new MockBaseTask(), taskReporter, Mockito.mock( + TaskMonitor taskMonitor = new TaskMonitor(Mockito.mock(TaskContainer.class), taskReporter, Mockito.mock( CloudObjectStorageService.class)); Assert.assertTrue(taskMonitor.reportTaskResultWithRetry(new DefaultTaskResult(), 3, 1)); } - - private static final class MockBaseTask implements Task { - - @Override - public void start(TaskContext taskContext) {} - - @Override - public boolean stop() { - return false; - } - - @Override - public boolean modify(Map jobParameters) { - return false; - } - - @Override - public double getProgress() { - return 0; - } - - @Override - public JobContext getJobContext() { - return null; - } - - @Override - public TaskStatus getStatus() { - return TaskStatus.RUNNING; - } - - @Override - public String getTaskResult() { - return "result"; - } - } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/Task.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/Task.java index 8be6e081e7..80665b34d0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/Task.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/Task.java @@ -17,9 +17,7 @@ import java.util.Map; -import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.enums.JobStatus; /** * Task interface. Each task should implement this interface @@ -30,14 +28,28 @@ public interface Task { /** - * Start current task. This method will be called by TaskExecutor for fire a task + * init task in runtime + * + * @param taskContext + * @throws Exception + */ + void init(TaskContext taskContext) throws Exception; + + /** + * Start current task. This method will be called by TaskExecutor for fire a task This method call + * must be blocked until job is done or failed */ - void start(TaskContext taskContext); + boolean start() throws Exception; /** * Stop current task. This method will be called TaskExecutor for stop a task */ - boolean stop(); + void stop() throws Exception; + + /** + * close and clean resource of task + */ + void close() throws Exception; /** * Modify current task parameters @@ -58,13 +70,6 @@ public interface Task { */ JobContext getJobContext(); - /** - * Get task status - * - * @return {@link JobStatus} - */ - TaskStatus getStatus(); - /** * Get task result */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskContext.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskContext.java index 497a3b703b..981ea06f30 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskContext.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskContext.java @@ -39,13 +39,6 @@ public interface TaskContext { */ JobContext getJobContext(); - /** - * provide task event listener - * - * @return - */ - TaskEventListener getTaskEventListener(); - /** * get shared storage for task upload or download file * diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/TaskBase.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/TaskBase.java new file mode 100644 index 0000000000..c2aafeca1f --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/TaskBase.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.base; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import com.oceanbase.odc.service.task.Task; +import com.oceanbase.odc.service.task.TaskContext; +import com.oceanbase.odc.service.task.caller.DefaultJobContext; +import com.oceanbase.odc.service.task.caller.JobContext; + +import lombok.extern.slf4j.Slf4j; + +/** + * base task for implement + * + * @author longpeng.zlp + * @date 2024/11/8 11:10 + */ +@Slf4j +public abstract class TaskBase implements Task { + protected TaskContext context; + protected DefaultJobContext jobContext; + + public void init(TaskContext taskContext) throws Exception { + this.context = taskContext; + jobContext = copyJobContext(taskContext); + log.info("Start task, id={}.", jobContext.getJobIdentity().getId()); + log.info("Init task parameters success, id={}.", jobContext.getJobIdentity().getId()); + doInit(jobContext); + } + + protected abstract void doInit(JobContext context) throws Exception; + + public JobContext getJobContext() { + return jobContext; + } + + // deep copy job context + protected DefaultJobContext copyJobContext(TaskContext taskContext) { + DefaultJobContext ret = new DefaultJobContext(); + JobContext src = taskContext.getJobContext(); + ret.setJobIdentity(src.getJobIdentity()); + if (null != src.getJobProperties()) { + ret.setJobProperties(new HashMap<>(src.getJobProperties())); + } + if (null != src.getJobParameters()) { + ret.setJobParameters(Collections.unmodifiableMap(new HashMap<>(src.getJobParameters()))); + } + ret.setJobClass(src.getJobClass()); + if (null != src.getHostUrls()) { + ret.setHostUrls(new ArrayList<>(src.getHostUrls())); + } + return ret; + } + + protected Long getJobId() { + return getJobContext().getJobIdentity().getId(); + } + + @Override + public boolean modify(Map jobParameters) { + if (Objects.isNull(jobParameters) || jobParameters.isEmpty()) { + log.warn("Job parameter cannot be null, id={}", getJobId()); + return false; + } + jobContext.setJobParameters(Collections.unmodifiableMap(jobParameters)); + return true; + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index 611fabe809..9e8a1b0760 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -35,8 +35,7 @@ import com.oceanbase.odc.service.dlm.utils.DlmJobIdUtil; import com.oceanbase.odc.service.schedule.job.DLMJobReq; import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; -import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.util.JobUtils; @@ -53,7 +52,7 @@ */ @Slf4j -public class DataArchiveTask extends BaseTask> { +public class DataArchiveTask extends TaskBase> { private DLMJobFactory jobFactory; private DLMJobStore jobStore; @@ -62,6 +61,7 @@ public class DataArchiveTask extends BaseTask> { private Map result; private boolean isToStop = false; + public DataArchiveTask() {} @Override protected void doInit(JobContext context) { @@ -71,11 +71,12 @@ protected void doInit(JobContext context) { } @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { + public boolean start() throws Exception { - jobStore.setJobParameters(getJobParameters()); + jobStore.setJobParameters(jobContext.getJobParameters()); DLMJobReq parameters = - JsonUtils.fromJson(getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), + JsonUtils.fromJson( + jobContext.getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), DLMJobReq.class); if (parameters.getFireTime() == null) { parameters.setFireTime(new Date()); @@ -86,7 +87,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex jobStore.setDlmTableUnits(result); } catch (Exception e) { log.warn("Get dlm job failed!", e); - taskContext.getExceptionListener().onException(e); + context.getExceptionListener().onException(e); return false; } Set dlmTableUnitIds = result.keySet(); @@ -94,7 +95,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex for (String dlmTableUnitId : dlmTableUnitIds) { DlmTableUnit dlmTableUnit = result.get(dlmTableUnitId); if (isToStop) { - log.info("Job is terminated,jobIdentity={}", context.getJobIdentity()); + log.info("Job is terminated,jobIdentity={}", jobContext.getJobIdentity()); break; } if (dlmTableUnit.getStatus() == TaskStatus.DONE) { @@ -138,7 +139,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex finishTableUnit(dlmTableUnitId, TaskStatus.CANCELED); } else { finishTableUnit(dlmTableUnitId, TaskStatus.FAILED); - taskContext.getExceptionListener().onException(e); + context.getExceptionListener().onException(e); } } } @@ -186,7 +187,7 @@ private List getDlmTableUnits(DLMJobReq req) throws SQLException { } @Override - protected void doStop() throws Exception { + public void stop() throws Exception { isToStop = true; if (job != null) { try { @@ -203,14 +204,22 @@ protected void doStop() throws Exception { } @Override - protected void doClose() throws Exception { + public void close() throws Exception { jobStore.destroy(); } @Override - protected void afterModifiedJobParameters() throws Exception { + public boolean modify(Map jobParameters) { + if (!super.modify(jobParameters)) { + return false; + } + afterModifiedJobParameters(); + return true; + } + + protected void afterModifiedJobParameters() { if (jobStore != null) { - jobStore.setJobParameters(getJobParameters()); + jobStore.setJobParameters(jobContext.getJobParameters()); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java index 56d0a504cf..25268324d1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java @@ -91,8 +91,7 @@ import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; import com.oceanbase.odc.service.session.initializer.ConsoleTimeoutInitializer; import com.oceanbase.odc.service.session.model.SqlExecuteResult; -import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.constants.JobServerUrls; @@ -113,7 +112,7 @@ */ @Slf4j -public class DatabaseChangeTask extends BaseTask { +public class DatabaseChangeTask extends TaskBase { private ConnectionSession connectionSession; private DatabaseChangeTaskParameters parameters; @@ -140,12 +139,15 @@ public class DatabaseChangeTask extends BaseTask { private volatile boolean canceled = false; private long taskId; + public DatabaseChangeTask() {} + @Override protected void doInit(JobContext jobContext) { taskId = getJobContext().getJobIdentity().getId(); log.info("Initiating database change task, taskId={}", taskId); - this.parameters = JobUtils.fromJson(getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), - DatabaseChangeTaskParameters.class); + this.parameters = + JobUtils.fromJson(jobContext.getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), + DatabaseChangeTaskParameters.class); this.databaseChangeParameters = JsonUtils.fromJson(this.parameters.getParameterJson(), DatabaseChangeParameters.class); log.info("Load database change task parameters successfully, taskId={}", taskId); @@ -186,7 +188,7 @@ protected void doInit(JobContext jobContext) { } @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws JobException { + public boolean start() throws JobException { try { int index = 0; while (sqlIterator.hasNext()) { @@ -244,7 +246,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Jo aborted = true; break; } - taskContext.getExceptionListener().onException(e); + context.getExceptionListener().onException(e); } } writeZipFile(); @@ -260,14 +262,14 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Jo } @Override - protected void doStop() { + public void stop() { tryExpireConnectionSession(); tryCloseInputStream(); canceled = true; } @Override - protected void doClose() throws Exception { + public void close() throws Exception { tryExpireConnectionSession(); tryCloseInputStream(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java index c6961e9b8b..eaa6723245 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java @@ -52,8 +52,7 @@ import com.oceanbase.odc.service.connection.logicaldatabase.model.DetailLogicalTableResp; import com.oceanbase.odc.service.schedule.model.PublishLogicalDatabaseChangeReq; import com.oceanbase.odc.service.session.model.SqlExecuteResult; -import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.tools.dbbrowser.parser.SqlParser; @@ -69,13 +68,15 @@ * @Description: [] */ @Slf4j -public class LogicalDatabaseChangeTask extends BaseTask>> { +public class LogicalDatabaseChangeTask extends TaskBase>> { private SqlRewriter sqlRewriter; private ExecutionGroupContext executionGroupContext; private PublishLogicalDatabaseChangeReq taskParameters; private List> executionGroups; private GroupExecutionEngine executorEngine; + public LogicalDatabaseChangeTask() {} + @Override protected void doInit(JobContext context) throws Exception { Map jobParameters = context.getJobParameters(); @@ -86,7 +87,7 @@ protected void doInit(JobContext context) throws Exception { } @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { + public boolean start() throws Exception { try { DialectType dialectType = taskParameters.getLogicalDatabaseResp().getDialectType(); DetailLogicalDatabaseResp detailLogicalDatabaseResp = taskParameters.getLogicalDatabaseResp(); @@ -178,7 +179,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex this.executionGroupContext = executorEngine.execute(executionGroups); } catch (Exception ex) { log.warn("start logical database change task failed, ", ex); - taskContext.getExceptionListener().onException(ex); + context.getExceptionListener().onException(ex); return false; } while (!Thread.currentThread().isInterrupted()) { @@ -188,7 +189,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex } if (CollectionUtils.isNotEmpty(this.executionGroupContext.getThrowables())) { log.warn("logical database change task failed, ", this.executionGroupContext.getThrowables()); - taskContext.getExceptionListener().onException(this.executionGroupContext.getThrowables().get(0)); + context.getExceptionListener().onException(this.executionGroupContext.getThrowables().get(0)); return false; } try { @@ -202,12 +203,12 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex } @Override - protected void doStop() throws Exception { + public void stop() { this.executorEngine.terminateAll(); } @Override - protected void doClose() throws Exception { + public void close() throws Exception { if (this.executorEngine != null) { this.executorEngine.close(); } @@ -224,8 +225,16 @@ public Map> getTaskResult() { } @Override - protected void afterModifiedJobParameters() throws Exception { - Map currentJobParameters = getJobParameters(); + public boolean modify(Map jobParameters) { + if (!super.modify(jobParameters)) { + return false; + } + afterModifiedJobParameters(); + return true; + } + + protected void afterModifiedJobParameters() { + Map currentJobParameters = jobContext.getJobParameters(); if (currentJobParameters == null || currentJobParameters.isEmpty()) { return; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java index cee3873bb0..d51ffb633b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java @@ -62,8 +62,7 @@ import com.oceanbase.odc.service.sqlcheck.SqlCheckRuleFactory; import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; import com.oceanbase.odc.service.sqlcheck.rule.SqlCheckRules; -import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.util.JobUtils; @@ -76,7 +75,7 @@ * @date 2024/1/30 11:02 */ @Slf4j -public class PreCheckTask extends BaseTask { +public class PreCheckTask extends TaskBase { private PreCheckTaskParameters parameters; private List userInputSqls; @@ -88,20 +87,22 @@ public class PreCheckTask extends BaseTask { private SqlCheckTaskResult sqlCheckResult = null; private DatabasePermissionCheckResult permissionCheckResult = null; + public PreCheckTask() {} + @Override protected void doInit(JobContext context) throws Exception { this.taskId = getJobContext().getJobIdentity().getId(); log.info("Initiating pre-check task, taskId={}", taskId); - this.parameters = JobUtils.fromJson(getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), - PreCheckTaskParameters.class); + this.parameters = + JobUtils.fromJson(jobContext.getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), + PreCheckTaskParameters.class); log.info("Load pre-check task parameters successfully, taskId={}", taskId); loadUserInputSqlContent(); loadUploadFileInputStream(); log.info("Load sql content successfully, taskId={}", taskId); } - @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { + public boolean start() throws Exception { try { List sqls = new ArrayList<>(); this.overLimit = getSqlContentUntilOverLimit(sqls, this.parameters.getMaxReadContentBytes()); @@ -118,7 +119,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex this.success = true; log.info("Pre-check task end up running, task id: {}", taskId); } catch (Throwable e) { - taskContext.getExceptionListener().onException(e); + context.getExceptionListener().onException(e); throw e; } finally { tryCloseInputStream(); @@ -127,12 +128,12 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex } @Override - protected void doStop() throws Exception { + public void stop() { tryCloseInputStream(); } @Override - protected void doClose() throws Exception { + public void close() { tryCloseInputStream(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java index dac0e87044..41c586d817 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java @@ -47,8 +47,7 @@ import com.oceanbase.odc.service.rollbackplan.UnsupportedSqlTypeForRollbackPlanException; import com.oceanbase.odc.service.rollbackplan.model.RollbackPlan; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; -import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.util.JobUtils; @@ -60,7 +59,7 @@ * @date 2024/2/6 11:03 */ @Slf4j -public class RollbackPlanTask extends BaseTask { +public class RollbackPlanTask extends TaskBase { private RollbackPlanTaskParameters parameters; private List userInputSqls; @@ -71,20 +70,22 @@ public class RollbackPlanTask extends BaseTask { private volatile boolean success = false; private volatile boolean aborted = false; + public RollbackPlanTask() {} + @Override protected void doInit(JobContext context) throws Exception { this.taskId = getJobContext().getJobIdentity().getId(); log.info("Initiating generate-rollback-plan task, taskId={}", taskId); - this.parameters = JobUtils.fromJson(getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), - RollbackPlanTaskParameters.class); + this.parameters = + JobUtils.fromJson(jobContext.getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), + RollbackPlanTaskParameters.class); log.info("Load generate-rollback-plan task parameters successfully, taskId={}", taskId); loadUserInputSqlContent(); loadUploadFileInputStream(); log.info("Load sql content successfully, taskId={}", taskId); } - @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { + public boolean start() throws Exception { try { long startTimeMills = System.currentTimeMillis(); ConnectionConfig connectionConfig = parameters.getConnectionConfig(); @@ -156,7 +157,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex } } catch (Exception e) { rollbackPlanTaskResult = RollbackPlanTaskResult.fail(e.getMessage()); - taskContext.getExceptionListener().onException(e); + context.getExceptionListener().onException(e); throw e; } finally { tryCloseInputStream(); @@ -165,13 +166,13 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex } @Override - protected void doStop() throws Exception { + public void stop() { this.aborted = true; tryCloseInputStream(); } @Override - protected void doClose() throws Exception { + public void close() { tryCloseInputStream(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java index 5d3ea7630f..a0e286b112 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java @@ -69,8 +69,7 @@ import com.oceanbase.odc.service.session.initializer.ConsoleTimeoutInitializer; import com.oceanbase.odc.service.session.model.SqlExecuteResult; import com.oceanbase.odc.service.sqlplan.model.SqlPlanTaskResult; -import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.exception.JobException; @@ -83,7 +82,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class SqlPlanTask extends BaseTask { +public class SqlPlanTask extends TaskBase { private PublishSqlPlanJobReq parameters; @@ -111,11 +110,14 @@ public class SqlPlanTask extends BaseTask { private final List csvFileMappers = new ArrayList<>(); + public SqlPlanTask() {} + @Override protected void doInit(JobContext context) { this.result = new SqlPlanTaskResult(); - this.parameters = JobUtils.fromJson(getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), - PublishSqlPlanJobReq.class); + this.parameters = + JobUtils.fromJson(jobContext.getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), + PublishSqlPlanJobReq.class); JobContext jobContext = getJobContext(); Map jobProperties = jobContext.getJobProperties(); this.taskId = jobContext.getJobIdentity().getId(); @@ -139,7 +141,7 @@ protected void doInit(JobContext context) { } @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { + public boolean start() throws Exception { try { int index = 0; initSqlInputStream(); @@ -244,14 +246,13 @@ private void initSqlInputStream() { } } - @Override - protected void doStop() { + public void stop() { canceled = true; } @Override - protected void doClose() { + public void close() { tryExpireConnectionSession(); } @@ -323,7 +324,7 @@ private boolean executeSqlWithRetries(String sql) { private ConnectionSession generateSession() { ConnectionConfig connectionConfig = JobUtils.fromJson( - getJobParameters().get(JobParametersKeyConstants.CONNECTION_CONFIG), ConnectionConfig.class); + jobContext.getJobParameters().get(JobParametersKeyConstants.CONNECTION_CONFIG), ConnectionConfig.class); DefaultConnectSessionFactory sessionFactory = new DefaultConnectSessionFactory(connectionConfig); sessionFactory.setSessionTimeoutMillis(parameters.getTimeoutMillis()); ConnectionSession connectionSession = sessionFactory.generateSession(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/DummyTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/DummyTask.java index c2be67a9da..8d77096c38 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/DummyTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/DummyTask.java @@ -18,8 +18,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.base.BaseTask; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import lombok.extern.slf4j.Slf4j; @@ -29,12 +28,14 @@ * @date 2024/8/28 10:51 */ @Slf4j -public class DummyTask extends BaseTask { +public class DummyTask extends TaskBase { private AtomicBoolean stopped = new AtomicBoolean(false); private AtomicLong loopCount = new AtomicLong(0); private final long maxLoopCount = 1000000; + public DummyTask() {} + @Override public double getProgress() { return (loopCount.get() / maxLoopCount) * 100; @@ -49,7 +50,7 @@ public String getTaskResult() { protected void doInit(JobContext context) throws Exception {} @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { + public boolean start() throws Exception { while (!stopped.get() && loopCount.get() < maxLoopCount) { Thread.sleep(1000); log.info("dummy task loop for to {}", loopCount.get()); @@ -58,12 +59,12 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex } @Override - protected void doStop() throws Exception { + public void stop() throws Exception { stopped.set(true); } @Override - protected void doClose() throws Exception { + public void close() throws Exception { stopped.set(true); } } diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java deleted file mode 100644 index 1e79e8b7ab..0000000000 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.task.executor.task; - -import java.util.HashMap; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.mockito.MockedStatic; -import org.mockito.Mockito; - -import com.oceanbase.odc.common.util.SystemUtils; -import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; -import com.oceanbase.odc.service.task.ExceptionListener; -import com.oceanbase.odc.service.task.TaskContext; -import com.oceanbase.odc.service.task.TaskEventListener; -import com.oceanbase.odc.service.task.base.BaseTask; -import com.oceanbase.odc.service.task.caller.DefaultJobContext; -import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; -import com.oceanbase.odc.service.task.executor.DefaultTaskResult; -import com.oceanbase.odc.service.task.executor.DefaultTaskResultBuilder; -import com.oceanbase.odc.service.task.schedule.JobIdentity; - -/** - * @author longpeng.zlp - * @date 2024/10/11 14:11 - */ -public class BaseTaskTest { - private DefaultJobContext jobContext; - - @Before - public void init() { - jobContext = new DefaultJobContext(); - jobContext.setJobParameters(new HashMap<>()); - jobContext.setJobProperties(new HashMap<>()); - JobIdentity jobIdentity = new JobIdentity(); - jobIdentity.setId(1L); - jobContext.setJobIdentity(jobIdentity); - jobContext.setJobClass(DummyBaseTask.class.getName()); - } - - - @Test - public void testExceptionListenerNormal() { - try (MockedStatic mockSystemUtil = Mockito.mockStatic(SystemUtils.class)) { - mockSystemUtil.when(() -> { - SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_EXECUTOR_PORT); - }).thenReturn("9099"); - DummyBaseTask dummyBaseTask = new DummyBaseTask(false); - DummyErrorListener dummyErrorListener = new DummyErrorListener(); - dummyBaseTask.start(new TaskContext() { - @Override - public ExceptionListener getExceptionListener() { - return dummyErrorListener; - } - - @Override - public JobContext getJobContext() { - return jobContext; - } - - @Override - public TaskEventListener getTaskEventListener() { - return Mockito.mock(TaskEventListener.class); - } - - @Override - public CloudObjectStorageService getSharedStorage() { - return Mockito.mock(CloudObjectStorageService.class); - } - }); - DefaultTaskResult taskResult = DefaultTaskResultBuilder.build(dummyBaseTask); - DefaultTaskResultBuilder.assignErrorMessage(taskResult, dummyErrorListener.error); - Assert.assertNull(taskResult.getErrorMessage()); - } - } - - @Test - public void testExceptionListenerWithException() { - try (MockedStatic mockSystemUtil = Mockito.mockStatic(SystemUtils.class)) { - mockSystemUtil.when(() -> { - SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_EXECUTOR_PORT); - }).thenReturn("9099"); - DummyBaseTask dummyBaseTask = new DummyBaseTask(true); - DummyErrorListener dummyErrorListener = new DummyErrorListener(); - dummyBaseTask.start(new TaskContext() { - @Override - public ExceptionListener getExceptionListener() { - return dummyErrorListener; - } - - @Override - public JobContext getJobContext() { - return jobContext; - } - - @Override - public TaskEventListener getTaskEventListener() { - return Mockito.mock(TaskEventListener.class); - } - - @Override - public CloudObjectStorageService getSharedStorage() { - return Mockito.mock(CloudObjectStorageService.class); - } - }); - DefaultTaskResult taskResult = DefaultTaskResultBuilder.build(dummyBaseTask); - DefaultTaskResultBuilder.assignErrorMessage(taskResult, dummyErrorListener.error); - Assert.assertEquals(taskResult.getErrorMessage(), "exception should be thrown"); - } - } - - private static final class DummyErrorListener implements ExceptionListener { - private Throwable error; - - @Override - public void onException(Throwable e) { - this.error = e; - } - } - - private static final class DummyBaseTask extends BaseTask { - private final boolean shouldThrowException; - - private DummyBaseTask(boolean shouldThrowException) { - this.shouldThrowException = shouldThrowException; - } - - @Override - protected void doInit(JobContext context) throws Exception {} - - @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { - if (shouldThrowException) { - throw new IllegalStateException("exception should be thrown"); - } - return true; - } - - @Override - protected void doStop() throws Exception {} - - @Override - protected void doClose() throws Exception {} - - @Override - public double getProgress() { - return 100; - } - - @Override - public String getTaskResult() { - return "res"; - } - } -} From e9cb4492307bbef7fde6b34cec17191770c3a50e Mon Sep 17 00:00:00 2001 From: LioRoger Date: Wed, 13 Nov 2024 15:10:04 +0800 Subject: [PATCH 026/118] feat(task): remove DefaultTaskResult (#3827) * remove DefaultTaskResult, fix TaskResult json deserialize failed * remove DefaultTaskResult, fix TaskResult json deserialize failed * remove DefaultTaskResult, fix TaskResult json deserialize failed --- .../oceanbase/odc/common/util/MapUtils.java | 31 ++++++++ .../odc/common/util/MapUtilsTest.java | 46 ++++++++++++ .../runtime/DefaultTaskResultBuilder.java | 8 +- .../agent/runtime/ExecutorRequestHandler.java | 6 +- .../odc/agent/runtime/TaskMonitor.java | 10 +-- .../web/controller/v2/TaskController.java | 4 +- .../odc/agent/runtime/TaskContainerTest.java | 7 +- .../odc/agent/runtime/TaskMonitorTest.java | 6 +- ...tabaseChangeRuntimeFlowableTaskCopied.java | 2 +- .../PreCheckRuntimeFlowableTaskCopied.java | 3 +- ...RollbackPlanRuntimeFlowableTaskCopied.java | 3 +- .../service/schedule/ScheduleTaskService.java | 15 +++- .../task/executor/DefaultTaskResult.java | 66 ---------------- .../odc/service/task/executor/TaskResult.java | 37 +++++++-- .../task/service/StdTaskFrameworkService.java | 11 ++- .../odc/service/task/util/JobUtils.java | 7 ++ .../service/task/util/TaskExecutorClient.java | 8 +- .../service/task/executor/TaskResultTest.java | 75 +++++++++++++++++++ 18 files changed, 233 insertions(+), 112 deletions(-) delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResult.java create mode 100644 server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/TaskResultTest.java diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/MapUtils.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/MapUtils.java index 854450d897..d10e8d044a 100644 --- a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/MapUtils.java +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/MapUtils.java @@ -18,6 +18,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.function.BiFunction; import java.util.stream.IntStream; import com.google.common.base.Splitter; @@ -128,4 +129,34 @@ public static int size(final Map map) { return org.apache.commons.collections4.MapUtils.size(map); } + /** + * compare map, null map and empty map consider as equals + * + * @param src + * @param target + * @param key of map + * @param value of map + * @param equalFunction function with argument value to compare if value instance is equals + * @return true is map is equals + */ + public static boolean isEqual(Map src, Map target, + BiFunction equalFunction) { + int currentSize = size(src); + int targetSize = size(target); + // map size not equals + if (currentSize != targetSize) { + return false; + } + if (currentSize == 0) { + return true; + } + // check key map + for (Map.Entry entry : src.entrySet()) { + if (!equalFunction.apply(entry.getValue(), target.get(entry.getKey()))) { + return false; + } + } + return true; + } + } diff --git a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/MapUtilsTest.java b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/MapUtilsTest.java index ad9b4981f6..fc90bcfad5 100644 --- a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/MapUtilsTest.java +++ b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/MapUtilsTest.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.TreeMap; import org.junit.Assert; import org.junit.Test; @@ -97,4 +98,49 @@ public void formatKvString_ValueContainsSeparator_IllegalArgumentException() { map.put("a", "2,"); MapUtils.formatKvString(map); } + + @Test + public void isEquals_emptyAndNull_equals() { + // empty and null is equals + Assert.assertTrue(MapUtils.isEqual(new HashMap(), null, String::equals)); + Assert.assertTrue(MapUtils.isEqual(null, null, String::equals)); + } + + @Test + public void isEquals_differentMapSize_notEquals() { + // size not equals + Assert.assertFalse(MapUtils.isEqual(new HashMap(), new TreeMap() { + { + put("key1", "value1"); + } + }, String::equals)); + } + + @Test + public void isEquals_sameValue_equals() { + // value equals + Assert.assertTrue(MapUtils.isEqual(new HashMap() { + { + put("key1", "value1"); + } + }, new TreeMap() { + { + put("key1", "value1"); + } + }, String::equals)); + } + + @Test + public void isEquals_differentValue_notEquals() { + // value not equals + Assert.assertFalse(MapUtils.isEqual(new HashMap() { + { + put("key1", "value1"); + } + }, new TreeMap() { + { + put("key1", null); + } + }, String::equals)); + } } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java index 6be6feadf5..5f1d32325a 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java @@ -17,7 +17,7 @@ import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.service.task.Task; -import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.util.JobUtils; /** @@ -27,8 +27,8 @@ */ class DefaultTaskResultBuilder { - public static DefaultTaskResult build(TaskContainer taskContainer) { - DefaultTaskResult result = new DefaultTaskResult(); + public static TaskResult build(TaskContainer taskContainer) { + TaskResult result = new TaskResult(); Task task = taskContainer.getTask(); result.setResultJson(JsonUtils.toJson(task.getTaskResult())); result.setStatus(taskContainer.getStatus()); @@ -38,7 +38,7 @@ public static DefaultTaskResult build(TaskContainer taskContainer) { return result; } - public static void assignErrorMessage(DefaultTaskResult result, Throwable e) { + public static void assignErrorMessage(TaskResult result, Throwable e) { result.setErrorMessage(null == e ? null : e.getMessage()); } } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java index 2079aa349e..89c257daf4 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java @@ -26,7 +26,7 @@ import com.oceanbase.odc.service.common.response.SuccessResponse; import com.oceanbase.odc.service.common.util.UrlUtils; import com.oceanbase.odc.service.task.constants.JobExecutorUrls; -import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.executor.logger.LogBiz; import com.oceanbase.odc.service.task.executor.logger.LogBizImpl; import com.oceanbase.odc.service.task.executor.logger.LogUtils; @@ -95,7 +95,7 @@ public SuccessResponse process(HttpMethod httpMethod, String uri, String JobIdentity ji = getJobIdentity(matcher); TaskRuntimeInfo runtimeInfo = ThreadPoolTaskExecutor.getInstance().getTaskRuntimeInfo(ji); TaskMonitor taskMonitor = runtimeInfo.getTaskMonitor(); - DefaultTaskResult result = DefaultTaskResultBuilder.build(runtimeInfo.getTaskContainer()); + TaskResult result = DefaultTaskResultBuilder.build(runtimeInfo.getTaskContainer()); if (taskMonitor != null && MapUtils.isNotEmpty(taskMonitor.getLogMetadata())) { result.setLogMetadata(taskMonitor.getLogMetadata()); // assign final error message @@ -103,7 +103,7 @@ public SuccessResponse process(HttpMethod httpMethod, String uri, String taskMonitor.markLogMetaCollected(); log.info("Task log metadata collected, ji={}.", ji.getId()); } - DefaultTaskResult copiedResult = ObjectUtil.deepCopy(result, DefaultTaskResult.class); + TaskResult copiedResult = ObjectUtil.deepCopy(result, TaskResult.class); return Responses.ok(copiedResult); } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java index b8ec027d39..05526a314e 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java @@ -35,8 +35,8 @@ import com.oceanbase.odc.service.task.constants.JobConstants; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.constants.JobServerUrls; -import com.oceanbase.odc.service.task.executor.DefaultTaskResult; import com.oceanbase.odc.service.task.executor.HeartbeatRequest; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.executor.TraceDecoratorThreadFactory; import com.oceanbase.odc.service.task.executor.logger.LogBizImpl; import com.oceanbase.odc.service.task.util.JobUtils; @@ -143,7 +143,7 @@ protected void reportTaskResult() { if (JobUtils.isReportDisabled()) { return; } - DefaultTaskResult taskResult = DefaultTaskResultBuilder.build(getTaskContainer()); + TaskResult taskResult = DefaultTaskResultBuilder.build(getTaskContainer()); if (taskResult.getStatus().isTerminated()) { log.info("job {} status {} is terminate, monitor report be ignored.", taskResult.getJobIdentity().getId(), taskResult.getStatus()); @@ -175,7 +175,7 @@ protected boolean isTimeout() { @VisibleForTesting protected void doFinal() { - DefaultTaskResult finalResult = DefaultTaskResultBuilder.build(getTaskContainer()); + TaskResult finalResult = DefaultTaskResultBuilder.build(getTaskContainer()); // Report final result log.info("Task id: {}, finished with status: {}, start to report final result", getJobId(), finalResult.getStatus()); @@ -222,7 +222,7 @@ protected void waitForTaskResultPulled() { } @VisibleForTesting - protected void uploadLogFileToCloudStorage(DefaultTaskResult finalResult) { + protected void uploadLogFileToCloudStorage(TaskResult finalResult) { Map logMap = finalResult.getLogMetadata(); Map logMetaData = new HashMap<>(); @@ -242,7 +242,7 @@ protected void uploadLogFileToCloudStorage(DefaultTaskResult finalResult) { } @VisibleForTesting - protected boolean reportTaskResultWithRetry(DefaultTaskResult result, int retries, int retryIntervalSeconds) { + protected boolean reportTaskResultWithRetry(TaskResult result, int retries, int retryIntervalSeconds) { if (result.getStatus() == TaskStatus.DONE) { result.setProgress(100.0); } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java index 6c9a44d271..7476b5685c 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java @@ -28,8 +28,8 @@ import com.oceanbase.odc.service.datasecurity.DataMaskingService; import com.oceanbase.odc.service.task.base.databasechange.QuerySensitiveColumnReq; import com.oceanbase.odc.service.task.base.databasechange.QuerySensitiveColumnResp; -import com.oceanbase.odc.service.task.executor.DefaultTaskResult; import com.oceanbase.odc.service.task.executor.HeartbeatRequest; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.service.TaskFrameworkService; import io.swagger.annotations.ApiOperation; @@ -53,7 +53,7 @@ public class TaskController { @ApiOperation(value = "updateResult", notes = "update task result") @RequestMapping(value = "/result", method = RequestMethod.POST) - public SuccessResponse updateResult(@RequestBody DefaultTaskResult taskResult) { + public SuccessResponse updateResult(@RequestBody TaskResult taskResult) { if (log.isDebugEnabled()) { log.debug("Accept task result {}.", JsonUtils.toJson(taskResult)); } diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java index 059608adc2..38f971709d 100644 --- a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java @@ -31,7 +31,7 @@ import com.oceanbase.odc.service.task.caller.DefaultJobContext; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; -import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.schedule.JobIdentity; /** @@ -63,7 +63,7 @@ public void testExceptionListenerNormal() { TaskContainer taskContainer = buildTaskContainer(jobContext, dummyBaseTask); taskContainer.runTask(); TaskReporter taskReporter = taskContainer.taskMonitor.getReporter(); - ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DefaultTaskResult.class); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskResult.class); Mockito.verify(taskReporter).report(ArgumentMatchers.any(), argumentCaptor.capture()); Assert.assertNull(argumentCaptor.getValue().getErrorMessage()); } @@ -79,8 +79,7 @@ public void testExceptionListenerWithException() { TaskContainer taskContainer = buildTaskContainer(jobContext, dummyBaseTask); taskContainer.runTask(); TaskReporter taskReporter = taskContainer.taskMonitor.getReporter(); - - ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DefaultTaskResult.class); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskResult.class); Mockito.verify(taskReporter).report(ArgumentMatchers.any(), argumentCaptor.capture()); Assert.assertEquals(argumentCaptor.getValue().getErrorMessage(), "exception should be thrown"); } diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java index c1fc08ca38..ccc96ca2fd 100644 --- a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java @@ -21,7 +21,7 @@ import org.mockito.Mockito; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; -import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; /** * @author longpeng.zlp @@ -35,7 +35,7 @@ public void testTaskMonitorReportRetryFailed() { Mockito.when(taskReporter.report(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); TaskMonitor taskMonitor = new TaskMonitor(Mockito.mock(TaskContainer.class), taskReporter, Mockito.mock( CloudObjectStorageService.class)); - Assert.assertFalse(taskMonitor.reportTaskResultWithRetry(new DefaultTaskResult(), 3, 1)); + Assert.assertFalse(taskMonitor.reportTaskResultWithRetry(new TaskResult(), 3, 1)); } @Test @@ -44,6 +44,6 @@ public void testTaskMonitorReportRetrySuccess() { Mockito.when(taskReporter.report(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(true); TaskMonitor taskMonitor = new TaskMonitor(Mockito.mock(TaskContainer.class), taskReporter, Mockito.mock( CloudObjectStorageService.class)); - Assert.assertTrue(taskMonitor.reportTaskResultWithRetry(new DefaultTaskResult(), 3, 1)); + Assert.assertTrue(taskMonitor.reportTaskResultWithRetry(new TaskResult(), 3, 1)); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java index d39455e8b6..41ec4c55a2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java @@ -148,7 +148,7 @@ protected DatabaseChangeResult start(Long taskId, TaskService taskService, Deleg TimeUnit.MILLISECONDS); JobEntity jobEntity = taskFrameworkService.find(this.jobId); - result = JsonUtils.fromJson(jobEntity.getResultJson(), DatabaseChangeResult.class); + result = JsonUtils.fromJson(JobUtils.retrieveJobResultStr(jobEntity), DatabaseChangeResult.class); result.setRollbackPlanResult(rollbackPlanTaskResult); if (jobEntity.getStatus() == JobStatus.DONE) { isSuccessful = true; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java index 3338a3aa26..c4d731ccfd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java @@ -141,7 +141,8 @@ protected Void start(Long taskId, TaskService taskService, DelegateExecution exe if (jobEntity.getStatus() != JobStatus.DONE) { throw new ServiceTaskError(new RuntimeException("Pre-check task failed")); } - this.preCheckResult = JsonUtils.fromJson(jobEntity.getResultJson(), PreCheckTaskResult.class); + this.preCheckResult = + JsonUtils.fromJson(JobUtils.retrieveJobResultStr(jobEntity), PreCheckTaskResult.class); if (Objects.nonNull(this.preCheckResult)) { this.preCheckResult.setExecutorInfo(new ExecutorInfo(this.hostProperties)); storeTaskResultToFile(this.preCheckResult.getSqlCheckResult()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java index e177c23308..d9b9797c16 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java @@ -94,7 +94,8 @@ protected RollbackPlanTaskResult start(Long taskId, TaskService taskService, Del if (Objects.isNull(jobEntity) || jobEntity.getStatus() != JobStatus.DONE) { throw new ServiceTaskError(new RuntimeException("Generate rollback plan task failed")); } - RollbackPlanTaskResult result = JsonUtils.fromJson(jobEntity.getResultJson(), RollbackPlanTaskResult.class); + RollbackPlanTaskResult result = + JsonUtils.fromJson(JobUtils.retrieveJobResultStr(jobEntity), RollbackPlanTaskResult.class); DatabaseChangeResult databaseChangeResult = new DatabaseChangeResult(); databaseChangeResult.setRollbackPlanResult(result); taskEntity.setResultJson(JsonUtils.toJson(databaseChangeResult)); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java index 2f570fa4c0..53ba9f63a8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java @@ -84,6 +84,7 @@ import com.oceanbase.odc.service.task.exception.JobException; import com.oceanbase.odc.service.task.model.ExecutorInfo; import com.oceanbase.odc.service.task.schedule.JobScheduler; +import com.oceanbase.odc.service.task.util.JobUtils; import lombok.extern.slf4j.Slf4j; @@ -154,7 +155,7 @@ public ScheduleTaskDetailResp getScheduleTaskDetailResp(Long id, Long scheduleId // sql plan task detail should display sql content res.setParameters(JsonUtils.toJson(scheduleTask.getParameters())); jobRepository.findByIdNative(scheduleTask.getJobId()) - .ifPresent(jobEntity -> res.setExecutionDetails(jobEntity.getResultJson())); + .ifPresent(jobEntity -> res.setExecutionDetails(JobUtils.retrieveJobResultStr(jobEntity))); break; case LOGICAL_DATABASE_CHANGE: res.setExecutionDetails( @@ -320,9 +321,15 @@ public Page getConditionalScheduleTaskListResp(Pageabl List jobIds = scheduleTaskPage.getContent().stream().map(ScheduleTask::getJobId).filter(Objects::nonNull) .collect(Collectors.toList()); - Map resultMap = jobRepository.findAllById(jobIds).stream() - .filter(jobEntity -> jobEntity.getResultJson() != null) - .collect(Collectors.toMap(JobEntity::getId, JobEntity::getResultJson)); + Map resultMap = new HashMap<>(); + List jobEntities = jobRepository.findAllById(jobIds); + // only not null result is collected + for (JobEntity jobEntity : jobEntities) { + String resultJson = JobUtils.retrieveJobResultStr(jobEntity); + if (null != resultJson) { + resultMap.put(jobEntity.getId(), resultJson); + } + } return scheduleTaskPage.map(task -> { Schedule schedule = scheduleMap.get(task.getJobName()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResult.java deleted file mode 100644 index d0edf02f17..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/DefaultTaskResult.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.task.executor; - -import java.util.Map; - -import com.oceanbase.odc.core.shared.constant.TaskStatus; -import com.oceanbase.odc.service.task.schedule.JobIdentity; - -import lombok.Data; - -/** - * @author yaobin - * @date 2023-11-29 - * @since 4.2.4 - */ -@Data -public class DefaultTaskResult implements TaskResult { - - private JobIdentity jobIdentity; - - private TaskStatus status; - - private String resultJson; - - private String executorEndpoint; - - private String errorMessage; - - private double progress; - - private Map logMetadata; - - public boolean progressChanged(DefaultTaskResult previous) { - if (previous == null) { - return true; - } - if (status != previous.getStatus()) { - return true; - } - if (Double.compare(progress, previous.getProgress()) != 0) { - return true; - } - if (logMetadata != null && !logMetadata.equals(previous.getLogMetadata())) { - return true; - } - if (resultJson != null && !resultJson.equals(previous.getResultJson())) { - return true; - } - return false; - } - -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java index 356a211cb0..df24042dfd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java @@ -17,27 +17,48 @@ import java.util.Map; +import com.oceanbase.odc.common.util.MapUtils; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.task.schedule.JobIdentity; +import lombok.Data; + /** * @author yaobin * @date 2023-11-29 * @since 4.2.4 */ -public interface TaskResult { +@Data +public class TaskResult { + + private JobIdentity jobIdentity; - JobIdentity getJobIdentity(); + private TaskStatus status; - TaskStatus getStatus(); + private String resultJson; - String getResultJson(); + private String executorEndpoint; - String getExecutorEndpoint(); + private String errorMessage; - String getErrorMessage(); + private double progress; - double getProgress(); + private Map logMetadata; - Map getLogMetadata(); + public boolean isProgressChanged(TaskResult previous) { + if (previous == null) { + return true; + } + if (status != previous.getStatus()) { + return true; + } + if (Double.compare(progress, previous.getProgress()) != 0) { + return true; + } + if (!MapUtils.isEqual(logMetadata, previous.logMetadata, String::equals)) { + return true; + } + return !StringUtils.equals(resultJson, previous.getResultJson()); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java index efcf69ec07..fc5b14c688 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java @@ -77,7 +77,6 @@ import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.enums.TaskRunMode; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.executor.DefaultTaskResult; import com.oceanbase.odc.service.task.executor.HeartbeatRequest; import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.listener.DefaultJobProcessUpdateEvent; @@ -359,18 +358,18 @@ private void doRefreshResult(Long id) throws JobException { } String executorEndpoint = executorEndpointManager.getExecutorEndpoint(je); - DefaultTaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); + TaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); if (result.getStatus() == TaskStatus.PREPARING) { log.info("Job is preparing, ignore refresh, jobId={}, currentStatus={}", id, result.getStatus()); return; } - DefaultTaskResult previous = JsonUtils.fromJson(je.getResultJson(), DefaultTaskResult.class); + TaskResult previous = JsonUtils.fromJson(je.getResultJson(), TaskResult.class); if (!updateHeartbeatTime(id)) { log.warn("Update lastHeartbeatTime failed, the job may finished or deleted already, jobId={}", id); return; } - if (!result.progressChanged(previous)) { + if (!result.isProgressChanged(previous)) { log.info("Progress not changed, skip update result to metadb, jobId={}, currentProgress={}", id, result.getProgress()); return; @@ -444,7 +443,7 @@ public boolean refreshLogMetaForCancelJob(Long id) { } try { String executorEndpoint = executorEndpointManager.getExecutorEndpoint(je); - DefaultTaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); + TaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); if (je.getRunMode().isK8s() && MapUtils.isEmpty(result.getLogMetadata())) { log.info("Refresh log failed due to log have not uploaded, jobId={}, currentStatus={}", je.getId(), @@ -514,7 +513,7 @@ private int updateExecutorEndpoint(Long id, String executorEndpoint, JobEntity c private int updateTaskResult(TaskResult taskResult, JobEntity currentJob, JobStatus expectedStatus) { JobEntity jse = new JobEntity(); handleTaskResult(currentJob.getJobType(), taskResult); - jse.setResultJson(taskResult.getResultJson()); + jse.setResultJson(JsonUtils.toJson(taskResult)); jse.setStatus(expectedStatus); jse.setProgressPercentage(taskResult.getProgress()); jse.setLastReportTime(JobDateUtils.getCurrentDate()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobUtils.java index 9f02bff787..e80ee3861f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobUtils.java @@ -30,12 +30,14 @@ import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.core.shared.Verify; import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; import com.oceanbase.odc.service.task.caller.JobEnvironmentEncryptor; import com.oceanbase.odc.service.task.constants.JobConstants; import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; import com.oceanbase.odc.service.task.enums.TaskRunMode; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.jasypt.AccessEnvironmentJasyptEncryptorConfigProperties; import com.oceanbase.odc.service.task.jasypt.DefaultJasyptEncryptor; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; @@ -188,4 +190,9 @@ public static String encrypt(String key, String salt, String raw) { public static String decrypt(String key, String salt, String encrypted) { return new JobEnvironmentEncryptor().decrypt(key, salt, encrypted); } + + public static String retrieveJobResultStr(JobEntity jobEntity) { + TaskResult taskResult = JsonUtils.fromJson(jobEntity.getResultJson(), TaskResult.class); + return null == taskResult ? null : taskResult.getResultJson(); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java index 3e375a6728..242f298062 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java @@ -28,7 +28,7 @@ import com.oceanbase.odc.service.schedule.ScheduleLogProperties; import com.oceanbase.odc.service.task.constants.JobExecutorUrls; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.executor.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.model.OdcTaskLogLevel; import com.oceanbase.odc.service.task.schedule.JobIdentity; @@ -110,12 +110,12 @@ public void modifyJobParameters(@NonNull String executorEndpoint, @NonNull JobId } } - public DefaultTaskResult getResult(@NonNull String executorEndpoint, @NonNull JobIdentity ji) throws JobException { + public TaskResult getResult(@NonNull String executorEndpoint, @NonNull JobIdentity ji) throws JobException { String url = executorEndpoint + String.format(JobExecutorUrls.GET_RESULT, ji.getId()); log.info("Try query job result from executor, jobId={}, url={}", ji.getId(), url); try { - SuccessResponse response = - HttpClientUtils.request("GET", url, new TypeReference>() {}); + SuccessResponse response = + HttpClientUtils.request("GET", url, new TypeReference>() {}); if (response != null && response.getSuccessful()) { return response.getData(); } else { diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/TaskResultTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/TaskResultTest.java new file mode 100644 index 0000000000..b0564f5b13 --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/TaskResultTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.executor; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +import org.junit.Assert; +import org.junit.Test; + +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.task.schedule.JobIdentity; + +/** + * @author longpeng.zlp + * @date 2024/11/11 15:09 + */ +public class TaskResultTest { + @Test + public void testTaskResult() { + Map logMetadata1 = new HashMap() { + { + put("key1", "value1"); + put("key2", "value2"); + } + }; + + Map logMetadata2 = new TreeMap() { + { + put("key1", null); + } + }; + + TaskResult result1 = createTaskResult(TaskStatus.RUNNING, 0.877, logMetadata1, "res1"); + Assert.assertFalse(result1.isProgressChanged(result1)); + Assert.assertTrue(result1.isProgressChanged(null)); + // log meta changed + TaskResult result2 = createTaskResult(TaskStatus.RUNNING, 0.877, logMetadata2, "res1"); + Assert.assertTrue(result1.isProgressChanged(result2)); + // statue changed + TaskResult result3 = createTaskResult(TaskStatus.DONE, 0.877, logMetadata1, "res1"); + Assert.assertTrue(result1.isProgressChanged(result3)); + // result json changed + TaskResult result4 = createTaskResult(TaskStatus.RUNNING, 0.877, logMetadata1, null); + Assert.assertTrue(result1.isProgressChanged(result4)); + // result progress changed + TaskResult result5 = createTaskResult(TaskStatus.RUNNING, 0.8774, logMetadata1, "res1"); + Assert.assertTrue(result1.isProgressChanged(result5)); + } + + private TaskResult createTaskResult(TaskStatus taskStatus, double progress, Map logMetadata, + String resultJson) { + TaskResult taskResult = new TaskResult(); + taskResult.setJobIdentity(JobIdentity.of(1L)); + taskResult.setResultJson(resultJson); + taskResult.setProgress(progress); + taskResult.setStatus(taskStatus); + taskResult.setLogMetadata(logMetadata); + return taskResult; + } +} From f8bffced9a586050a7301cdf774c7a56d43ffab7 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Wed, 13 Nov 2024 15:54:55 +0800 Subject: [PATCH 027/118] feat(external tables): supports syncing external table files (#3802) * finish framework of syncing external table files * Implement external table file sync in OBMySQL and OBOracle * Refactor external table file sync response handling * Refactor external table file sync to return boolean --- .../tools/dbbrowser/schema/DBSchemaAccessor.java | 6 ++++++ .../dbbrowser/schema/doris/DorisSchemaAccessor.java | 5 +++++ .../mysql/MySQLNoLessThan5700SchemaAccessor.java | 5 +++++ .../schema/mysql/OBMySQLSchemaAccessor.java | 10 +++++++++- .../OBOracleBetween410And432SchemaAccessor.java | 5 +++++ .../schema/oracle/OBOracleSchemaAccessor.java | 8 ++++++++ .../schema/oracle/OracleSchemaAccessor.java | 5 +++++ .../schema/postgre/PostgresSchemaAccessor.java | 5 +++++ .../server/web/controller/v2/DBTableController.java | 10 ++++++++++ .../com/oceanbase/odc/service/db/DBTableService.java | 12 ++++++++++++ .../odc/plugin/schema/api/TableExtensionPoint.java | 2 ++ .../odc/plugin/schema/mysql/MySQLTableExtension.java | 5 +++++ .../plugin/schema/obmysql/OBMySQLTableExtension.java | 7 +++++++ .../obmysql/ODPShardingOBMySQLTableExtension.java | 5 +++++ .../plugin/schema/oracle/OracleTableExtension.java | 5 +++++ .../schema/postgres/PostgresTableExtension.java | 5 ++++- 16 files changed, 98 insertions(+), 2 deletions(-) diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java index 64439fb2f6..16a6a2ad27 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java @@ -101,6 +101,12 @@ default List showExternalTables(String schemaName) { */ boolean isExternalTable(String schemaName, String tableName); + /** + * Synchronize the associated files of external table + */ + boolean syncExternalTableFiles(String schemaName, String tableName); + + /** * Show all view names list in the specified schema */ diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java index 0842716d8e..d2b9bdfaa7 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java @@ -240,6 +240,11 @@ public boolean isExternalTable(String schemaName, String tableName) { return false; } + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + protected List listBaseTables(String schemaName, String tableNameLike) throws DataAccessException { MySQLSqlBuilder sb = new MySQLSqlBuilder(); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java index b0f1d84c9a..d5a87e7e85 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java @@ -254,6 +254,11 @@ public boolean isExternalTable(String schemaName, String tableName) { return false; } + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + protected List listBaseTables(String schemaName, String tableNameLike) throws DataAccessException { MySQLSqlBuilder sb = new MySQLSqlBuilder(); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java index 1ef531c7e7..e46d42073a 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java @@ -60,7 +60,7 @@ @Slf4j public class OBMySQLSchemaAccessor extends MySQLNoLessThan5700SchemaAccessor { - protected static final Set ESCAPE_SCHEMA_SET = new HashSet<>(3); + protected static final Set ESCAPE_SCHEMA_SET = new HashSet<>(4); static { ESCAPE_SCHEMA_SET.add("PUBLIC"); @@ -424,6 +424,14 @@ public boolean isExternalTable(String schemaName, String tableName) { } } + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + MySQLSqlBuilder sb = new MySQLSqlBuilder(); + sb.append("ALTER EXTERNAL TABLE ").identifier(schemaName, tableName).append(" REFRESH"); + jdbcOperations.execute(sb.toString()); + return true; + } + @Override protected void correctColumnPrecisionIfNeed(List tableColumns) {} } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java index 2996ba695b..742232d444 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java @@ -72,6 +72,11 @@ public List showTablesLike(String schemaName, String tableNameLike) { return jdbcOperations.queryForList(sb.toString(), String.class); } + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + @Override public List listTables(String schemaName, String tableNameLike) { OracleSqlBuilder sb = new OracleSqlBuilder(); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java index b42a09796b..e0dd96a7d8 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java @@ -1069,6 +1069,14 @@ public List listExternalTables(String schemaName, String table return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); } + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("ALTER EXTERNAL TABLE ").identifier(schemaName, tableName).append(" REFRESH"); + jdbcOperations.execute(sb.toString()); + return true; + } + // After ob version 4.3.2, oracle model displaying table list needs to exclude external tables @Override public List showTablesLike(String schemaName, String tableNameLike) { diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java index 4e787b63cc..0cb247c93d 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java @@ -262,6 +262,11 @@ public boolean isExternalTable(String schemaName, String tableName) { return false; } + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + @Override public List listViews(String schemaName) { OracleSqlBuilder sb = new OracleSqlBuilder(); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java index e75a823769..637d26a651 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java @@ -152,6 +152,11 @@ public boolean isExternalTable(String schemaName, String tableName) { return false; } + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + @Override public List listViews(String schemaName) { throw new UnsupportedOperationException("Not supported yet"); diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java index 8bbfe38344..2cf205b45d 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java @@ -122,4 +122,14 @@ public ListResponse getPartitionKeyDataTypes(@PathVariable String sess return Responses.list(this.partitionPlanService.getPartitionKeyDataTypes(sessionId, databaseId, tableName)); } + @PostMapping(value = "/{sessionId}/databases/{databaseName}/externalTables/" + + "{externalTableName}/syncExternalTableFiles") + public SuccessResponse syncExternalTableFiles(@PathVariable String sessionId, + @PathVariable(required = false) String databaseName, + @PathVariable(required = true, name = "externalTableName") String externalTableName) { + Base64.Decoder decoder = Base64.getDecoder(); + externalTableName = new String(decoder.decode(externalTableName)); + ConnectionSession session = sessionService.nullSafeGet(sessionId, true); + return Responses.success(tableService.syncExternalTableFiles(session, databaseName, externalTableName)); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java index ace6aa5693..2ba8ba1d50 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java @@ -183,6 +183,18 @@ public GenerateTableDDLResp generateUpdateDDL(@NotNull ConnectionSession session .build(); } + public boolean syncExternalTableFiles(@NotNull ConnectionSession connectionSession, String schemaName, + @NotBlank String externalTableName) { + DBSchemaAccessor schemaAccessor = DBSchemaAccessors.create(connectionSession); + PreConditions.validExists(ResourceType.OB_TABLE, "tableName", externalTableName, + () -> schemaAccessor.showExternalTables(schemaName).stream() + .filter(name -> name.equals(externalTableName)) + .collect(Collectors.toList()).size() > 0); + return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) + .execute((ConnectionCallback) con -> getTableExtensionPoint(connectionSession) + .syncExternalTableFiles(con, schemaName, externalTableName)); + } + private String checkUpdateDDL(DialectType dialectType, String ddl) { boolean createIndex = false; boolean dropIndex = false; diff --git a/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java b/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java index 25db79b650..2adc68b617 100644 --- a/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java +++ b/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java @@ -44,4 +44,6 @@ public interface TableExtensionPoint extends ExtensionPoint { String generateCreateDDL(Connection connection, DBTable table); String generateUpdateDDL(Connection connection, DBTable oldTable, DBTable newTable); + + boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName); } diff --git a/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java b/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java index f8a14973ce..0488dd4467 100644 --- a/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java +++ b/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java @@ -74,4 +74,9 @@ protected DBStatsAccessor getStatsAccessor(Connection connection) { return DBAccessorUtil.getStatsAccessor(connection); } + @Override + public boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName) { + throw new UnsupportedOperationException("not implemented yet"); + } + } diff --git a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java index c3fd04f1a3..62801657bd 100644 --- a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java +++ b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java @@ -138,6 +138,13 @@ public String generateUpdateDDL(@NonNull Connection connection, @NonNull DBTable return getTableEditor(connection).generateUpdateObjectDDL(oldTable, newTable); } + @Override + public boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName) { + DBSchemaAccessor schemaAccessor = getSchemaAccessor(connection); + schemaAccessor.syncExternalTableFiles(schemaName, tableName); + return true; + } + protected DBTableEditor getTableEditor(Connection connection) { return DBAccessorUtil.getTableEditor(connection); } diff --git a/server/plugins/schema-plugin-odp-sharding-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/odpsharding/obmysql/ODPShardingOBMySQLTableExtension.java b/server/plugins/schema-plugin-odp-sharding-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/odpsharding/obmysql/ODPShardingOBMySQLTableExtension.java index 911b0aede4..0a4bb492f5 100644 --- a/server/plugins/schema-plugin-odp-sharding-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/odpsharding/obmysql/ODPShardingOBMySQLTableExtension.java +++ b/server/plugins/schema-plugin-odp-sharding-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/odpsharding/obmysql/ODPShardingOBMySQLTableExtension.java @@ -63,4 +63,9 @@ protected DBTableEditor getTableEditor(Connection connection) { .setType(DialectType.ODP_SHARDING_OB_MYSQL.getDBBrowserDialectTypeName()).create(); } + @Override + public boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName) { + throw new UnsupportedOperationException("not implemented yet"); + } + } diff --git a/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java b/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java index 1c05686373..e3eccc2747 100644 --- a/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java +++ b/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java @@ -62,6 +62,11 @@ public DBTable getDetail(@NonNull Connection connection, @NonNull String schemaN return table; } + @Override + public boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName) { + throw new UnsupportedOperationException("not implemented yet"); + } + @Override protected DBTableStats getTableStats(@NonNull Connection connection, @NonNull String schemaName, @NonNull String tableName) { diff --git a/server/plugins/schema-plugin-postgres/src/main/java/com/oceanbase/odc/plugin/schema/postgres/PostgresTableExtension.java b/server/plugins/schema-plugin-postgres/src/main/java/com/oceanbase/odc/plugin/schema/postgres/PostgresTableExtension.java index 68d31c186f..d2ee8bf4cf 100644 --- a/server/plugins/schema-plugin-postgres/src/main/java/com/oceanbase/odc/plugin/schema/postgres/PostgresTableExtension.java +++ b/server/plugins/schema-plugin-postgres/src/main/java/com/oceanbase/odc/plugin/schema/postgres/PostgresTableExtension.java @@ -31,5 +31,8 @@ protected DBSchemaAccessor getSchemaAccessor(Connection connection) { return DBAccessorUtil.getSchemaAccessor(connection); } - + @Override + public boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName) { + throw new UnsupportedOperationException("not implemented yet"); + } } From 657b7c136a70678b988c184e160dbc3561c9e3e4 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Wed, 13 Nov 2024 19:26:13 +0800 Subject: [PATCH 028/118] feat(sql-console-rule): sql console adapt sql type of call ,comment on and set session (#3764) --- .../tools/dbbrowser/parser/ParserUtil.java | 1 + .../dbbrowser/parser/constant/SqlType.java | 56 +++--- .../listener/MysqlModePLParserListener.java | 7 + .../listener/MysqlModeSqlParserListener.java | 1 + .../listener/OracleModePLParserListener.java | 6 + .../listener/OracleModeParserListener.java | 35 +++- .../listener/OracleModeSqlParserListener.java | 24 ++- .../tools/dbbrowser/parser/PLParserTest.java | 50 ++++++ .../dbbrowser/parser/ParserUtilTest.java | 92 ++++++++++ .../tools/dbbrowser/parser/SqlParserTest.java | 75 ++++++++ .../init/regulation-rule-metadata.yaml | 170 +++++++++--------- .../interceptor/SqlConsoleInterceptor.java | 11 +- .../session/model/SqlExecuteResult.java | 3 +- 13 files changed, 425 insertions(+), 106 deletions(-) diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/ParserUtil.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/ParserUtil.java index 5611084cbf..118b4c4caa 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/ParserUtil.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/ParserUtil.java @@ -55,6 +55,7 @@ public static GeneralSqlType getGeneralSqlType(BasicResult result) { case DROP: case TRUNCATE: case ALTER: + case COMMENT_ON: return GeneralSqlType.DDL; case UNKNOWN: default: diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/constant/SqlType.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/constant/SqlType.java index f7148310fe..56e743f5e8 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/constant/SqlType.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/constant/SqlType.java @@ -19,25 +19,39 @@ * Created by mogao.zj */ public enum SqlType { - SELECT, - DELETE, - INSERT, - REPLACE, - UPDATE, - SET, - USE_DB, - EXPLAIN, - SHOW, - HELP, - START_TRANS, - COMMIT, - ROLLBACK, - SORT, - DESC, - DROP, - ALTER, - TRUNCATE, - CREATE, - OTHERS, - UNKNOWN + SELECT(null), + DELETE(null), + INSERT(null), + REPLACE(null), + UPDATE(null), + SET(null), + SET_SESSION(SET), + USE_DB(null), + EXPLAIN(null), + SHOW(null), + HELP(null), + START_TRANS(null), + COMMIT(null), + ROLLBACK(null), + SORT(null), + DESC(null), + DROP(null), + ALTER(null), + ALTER_SESSION(ALTER), + TRUNCATE(null), + CREATE(null), + CALL(null), + COMMENT_ON(null), + OTHERS(null), + UNKNOWN(null); + + private final SqlType parentType; + + SqlType(SqlType parentType) { + this.parentType = parentType; + } + + public SqlType getParentType() { + return parentType; + } } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModePLParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModePLParserListener.java index 8c59c2ad68..5fd374b743 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModePLParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModePLParserListener.java @@ -159,4 +159,11 @@ public void enterSp_decl(Sp_declContext ctx) { varibale.setVarType(type); this.varibaleList.add(varibale); } + + @Override + public void enterCall_sp_stmt(PLParser.Call_sp_stmtContext ctx) { + this.sqlType = SqlType.CALL; + this.dbObjectType = DBObjectType.PROCEDURE; + } + } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java index 40f3745470..f5ab1727db 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java @@ -228,6 +228,7 @@ public void enterScope_or_scope_alias(Scope_or_scope_aliasContext ctx) { if (scopeToken.getType() == OBLexer.GLOBAL || scopeToken.getType() == OBLexer.GLOBAL_ALIAS) { setDbObjectType(DBObjectType.GLOBAL_VARIABLE); } else if (scopeToken.getType() == OBLexer.SESSION || scopeToken.getType() == OBLexer.SESSION_ALIAS) { + this.sqlType = SqlType.SET_SESSION; setDbObjectType(DBObjectType.SESSION_VARIABLE); } } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModePLParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModePLParserListener.java index 7568f3da48..77474ec04d 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModePLParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModePLParserListener.java @@ -555,6 +555,12 @@ public void enterAlter_package_stmt(Alter_package_stmtContext ctx) { } } + @Override + public void enterCall_spec(PLParser.Call_specContext ctx) { + this.sqlType = sqlType.CALL; + this.dbObjectType = DBObjectType.PROCEDURE; + } + private String getDdl(ParserRuleContext ctx) { Token start = ctx.getStart(); Token stop = ctx.getStop(); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java index d0dc419617..e20bb82060 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java @@ -68,7 +68,6 @@ import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Procedure_bodyContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Procedure_nameContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Procedure_specContext; -import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Rollback_statementContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Type_declarationContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Variable_declarationContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParserBaseListener; @@ -418,6 +417,28 @@ public void enterRollback_statement(PlSqlParser.Rollback_statementContext ctx) { doRecord(SqlType.ROLLBACK, DBObjectType.OTHERS, null); } + @Override + public void enterComment_on_column(PlSqlParser.Comment_on_columnContext ctx) { + doRecord(SqlType.COMMENT_ON, DBObjectType.COLUMN, null); + } + + @Override + public void enterComment_on_table(PlSqlParser.Comment_on_tableContext ctx) { + doRecord(SqlType.COMMENT_ON, DBObjectType.TABLE, null); + } + + @Override + public void enterComment_on_materialized(PlSqlParser.Comment_on_materializedContext ctx) { + doRecord(SqlType.COMMENT_ON, DBObjectType.OTHERS, null); + } + + @Override + public void enterProcedure_call(PlSqlParser.Procedure_callContext ctx) { + if (ctx.CALL() != null) { + doRecord(SqlType.CALL, DBObjectType.PROCEDURE, null); + } + } + private void doRecord(SqlType sqlType, DBObjectType dbObjectType, String rawPlName) { if (Objects.nonNull(sqlType) && Objects.isNull(this.sqlType)) { this.sqlType = sqlType; @@ -431,6 +452,18 @@ private void doRecord(SqlType sqlType, DBObjectType dbObjectType, String rawPlNa } } + @Override + public void enterAlter_session(PlSqlParser.Alter_sessionContext ctx) { + doRecord(SqlType.ALTER_SESSION, DBObjectType.OTHERS, null); + } + + @Override + public void enterScope_or_scope_alias(PlSqlParser.Scope_or_scope_aliasContext ctx) { + if (ctx.SESSION() != null) { + doRecord(SqlType.SET_SESSION, DBObjectType.OTHERS, null); + } + } + private String getDdl(ParserRuleContext ctx) { Token start = ctx.getStart(); Token stop = ctx.getStop(); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java index 0ef7c736ac..e6274c7ea4 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java @@ -284,6 +284,7 @@ public void enterScope_or_scope_alias(Scope_or_scope_aliasContext ctx) { if (scopeToken.getType() == OBLexer.GLOBAL || scopeToken.getType() == OBLexer.GLOBAL_ALIAS) { setDbObjectType(DBObjectType.GLOBAL_VARIABLE); } else if (scopeToken.getType() == OBLexer.SESSION || scopeToken.getType() == OBLexer.SESSION_ALIAS) { + sqlType = sqlType.SET_SESSION; setDbObjectType(DBObjectType.SESSION_VARIABLE); } } @@ -601,7 +602,7 @@ public void enterAlter_sequence_stmt(Alter_sequence_stmtContext ctx) { @Override public void enterAlter_session_stmt(Alter_session_stmtContext ctx) { - setSqlType(SqlType.ALTER); + setSqlType(SqlType.ALTER_SESSION); this.dbObjectType = DBObjectType.OTHERS; } @@ -700,4 +701,25 @@ public void enterRollback_stmt(Rollback_stmtContext ctx) { this.dbObjectType = DBObjectType.OTHERS; } + @Override + public void enterSet_comment_stmt(OBParser.Set_comment_stmtContext ctx) { + // comment on xxx + setSqlType(SqlType.COMMENT_ON); + if (ctx.TABLE() != null) { + this.dbObjectType = DBObjectType.TABLE; + return; + } + if (ctx.COLUMN() != null) { + this.dbObjectType = DBObjectType.COLUMN; + return; + } + this.dbObjectType = DBObjectType.OTHERS; + } + + @Override + public void enterCall_stmt(OBParser.Call_stmtContext ctx) { + setSqlType(SqlType.CALL); + this.dbObjectType = DBObjectType.PROCEDURE; + } + } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java index 5aded51d8a..1e2114b9c9 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java @@ -1183,4 +1183,54 @@ public void parseOracle_rollbackStmt_getSqlTypeSucceed() { Assert.assertEquals(SqlType.ROLLBACK, actual.getSqlType()); } + @Test + public void parseOracle_commentOnTable_getSqlTypeSucceed() { + ParseOraclePLResult actual = PLParser.parseOracle("comment on table a is 'xxx'"); + Assert.assertEquals(DBObjectType.TABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMENT_ON, actual.getSqlType()); + } + + @Test + public void parseOracle_commentOnColumn_getSqlTypeSucceed() { + ParseOraclePLResult actual = PLParser.parseOracle("comment on column a is 'xxx'"); + Assert.assertEquals(DBObjectType.COLUMN, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMENT_ON, actual.getSqlType()); + } + + @Test + public void parseOracle_commentOnMaterialized_getSqlTypeSucceed() { + ParseOraclePLResult actual = PLParser.parseOracle("comment on materialized a is 'xxx'"); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMENT_ON, actual.getSqlType()); + } + + @Test + public void parseOracle_call_getSqlTypeSucceed() { + ParseOraclePLResult actual = PLParser.parseOracle("call proc()"); + Assert.assertEquals(DBObjectType.PROCEDURE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.CALL, actual.getSqlType()); + } + + @Test + public void parseOBMysql_call_getSqlTypeSucceed() { + ParseMysqlPLResult actual = PLParser.parseObMysql("call proc()"); + Assert.assertEquals(DBObjectType.PROCEDURE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.CALL, actual.getSqlType()); + } + + @Test + public void test_oracle_alter_session() { + String sql = "alter SESSION set ob_query_timeout=6000000000;"; + ParseOraclePLResult result = PLParser.parseOracle(sql); + Assert.assertEquals(SqlType.ALTER_SESSION, result.getSqlType()); + Assert.assertEquals(DBObjectType.OTHERS, result.getDbObjectType()); + } + + @Test + public void test_oracle_set_session() { + String sql = "SET SESSION ob_query_timeout=6000000000;"; + ParseOraclePLResult result = PLParser.parseOracle(sql); + Assert.assertEquals(SqlType.SET_SESSION, result.getSqlType()); + Assert.assertEquals(DBObjectType.OTHERS, result.getDbObjectType()); + } } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/ParserUtilTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/ParserUtilTest.java index 06d00c1d56..4a51bb61c2 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/ParserUtilTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/ParserUtilTest.java @@ -288,4 +288,96 @@ public void test_mysql_pl_syntax_error() { BasicResult result = ParserUtil.parseOracleType(sql); Assert.assertTrue(result.getSyntaxError()); } + + @Test + public void test_mysql_set_session() { + String sql = "SET SESSION time_zone = '+00:00';"; + BasicResult result = ParserUtil.parseMysqlType(sql); + Assert.assertEquals(SqlType.SET_SESSION, result.getSqlType()); + Assert.assertEquals(SqlType.SET, result.getSqlType().getParentType()); + Assert.assertEquals(DBObjectType.SESSION_VARIABLE, result.getDbObjectType()); + } + + @Test + public void test_mysql_set_global() { + String sql = "SET GLOBAL time_zone = '+00:00';"; + BasicResult result = ParserUtil.parseMysqlType(sql); + Assert.assertEquals(SqlType.SET, result.getSqlType()); + Assert.assertEquals(DBObjectType.GLOBAL_VARIABLE, result.getDbObjectType()); + } + + @Test + public void test_oracle_set_global() { + String sql = "SET GLOBAL time_zone = '+00:00';"; + BasicResult result = ParserUtil.parseMysqlType(sql); + Assert.assertEquals(SqlType.SET, result.getSqlType()); + Assert.assertEquals(DBObjectType.GLOBAL_VARIABLE, result.getDbObjectType()); + } + + @Test + public void test_oracle_alter_system() { + String sql = "ALTER SYSTEM SET time_zone = '+00:00';"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.ALTER, result.getSqlType()); + Assert.assertEquals(DBObjectType.OTHERS, result.getDbObjectType()); + } + + @Test + public void test_mysql_set() { + String sql = "SET time_zone = '+00:00';"; + BasicResult result = ParserUtil.parseMysqlType(sql); + Assert.assertEquals(SqlType.SET, result.getSqlType()); + Assert.assertEquals(DBObjectType.SYSTEM_VARIABLE, result.getDbObjectType()); + } + + @Test + public void test_mysql_call() { + String sql = "call proc()"; + BasicResult result = ParserUtil.parseMysqlType(sql); + Assert.assertEquals(SqlType.CALL, result.getSqlType()); + Assert.assertEquals(DBObjectType.PROCEDURE, result.getDbObjectType()); + } + + @Test + public void test_oracle_call() { + String sql = "call proc()"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.CALL, result.getSqlType()); + Assert.assertEquals(DBObjectType.PROCEDURE, result.getDbObjectType()); + } + + @Test + public void test_oracle_alterSession() { + String sql = "alter SESSION set ob_query_timeout=6000000000;"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.ALTER_SESSION, result.getSqlType()); + Assert.assertEquals(SqlType.ALTER, result.getSqlType().getParentType()); + Assert.assertEquals(DBObjectType.OTHERS, result.getDbObjectType()); + } + + @Test + public void test_oracle_setSession() { + String sql = "set SESSION ob_query_timeout=6000000000;"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.SET_SESSION, result.getSqlType()); + Assert.assertEquals(SqlType.SET, result.getSqlType().getParentType()); + Assert.assertEquals(DBObjectType.SESSION_VARIABLE, result.getDbObjectType()); + } + + @Test + public void test_oracle_comment_on_table() { + String sql = "comment on table t is 'abc'"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.COMMENT_ON, result.getSqlType()); + Assert.assertEquals(DBObjectType.TABLE, result.getDbObjectType()); + } + + @Test + public void test_oracle_comment_on_column() { + String sql = "comment on column t is 'abc'"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.COMMENT_ON, result.getSqlType()); + Assert.assertEquals(DBObjectType.COLUMN, result.getDbObjectType()); + } + } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java index 006996da61..6e938d1898 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java @@ -284,4 +284,79 @@ public void parseOracle_rollbackStmt_getSqlTypeSucceed() { Assert.assertEquals(SqlType.ROLLBACK, actual.getSqlType()); } + @Test + public void parseOracle_commentOnTable_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("comment on table a is 'xxx'"); + Assert.assertEquals(DBObjectType.TABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMENT_ON, actual.getSqlType()); + } + + @Test + public void parseOracle_commentOnColumn_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("comment on column a is 'xxx'"); + Assert.assertEquals(DBObjectType.COLUMN, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMENT_ON, actual.getSqlType()); + } + + @Test + public void parseOracle_call_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("call proc()"); + Assert.assertEquals(DBObjectType.PROCEDURE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.CALL, actual.getSqlType()); + } + + + @Test + public void parseMysql_setSession_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseMysql("SET SESSION time_zone = '+00:00';"); + Assert.assertEquals(DBObjectType.SESSION_VARIABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.SET_SESSION, actual.getSqlType()); + Assert.assertEquals(SqlType.SET, actual.getSqlType().getParentType()); + } + + @Test + public void parseMysql_setGlobal_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseMysql("SET GLOBAL time_zone = '+00:00';"); + Assert.assertEquals(DBObjectType.GLOBAL_VARIABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.SET, actual.getSqlType()); + } + + @Test + public void parseMysql_set_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseMysql("SET time_zone = '+00:00';"); + Assert.assertEquals(DBObjectType.SYSTEM_VARIABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.SET, actual.getSqlType()); + } + + @Test + public void parseOracle_alterSession_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("alter SESSION set ob_query_timeout=6000000000;"); + Assert.assertEquals(SqlType.ALTER_SESSION, actual.getSqlType()); + Assert.assertEquals(SqlType.ALTER, actual.getSqlType().getParentType()); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + } + + @Test + public void parseOracle_setSession_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("set SESSION ob_query_timeout=6000000000;"); + Assert.assertEquals(SqlType.SET_SESSION, actual.getSqlType()); + Assert.assertEquals(SqlType.SET, actual.getSqlType().getParentType()); + Assert.assertEquals(DBObjectType.SESSION_VARIABLE, actual.getDbObjectType()); + } + + @Test + public void parseOracle_setGlobal_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("SET GLOBAL time_zone = '+00:00';"); + Assert.assertEquals(DBObjectType.GLOBAL_VARIABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.SET, actual.getSqlType()); + } + + @Test + public void parseOracle_alterSystem_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("alter system set time_zone = '+00:00';"); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + Assert.assertEquals(SqlType.ALTER, actual.getSqlType()); + } + + } diff --git a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml index d519181ab1..5887c63922 100644 --- a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml +++ b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml @@ -94,8 +94,10 @@ - CREATE - DROP - ALTER + - ALTER_SESSION - REPLACE - SET + - SET_SESSION - USE_DB - EXPLAIN - SHOW @@ -106,6 +108,8 @@ - SORT - DESC - TRUNCATE + - CALL + - COMMENT_ON - OTHERS candidates: - UPDATE @@ -115,8 +119,10 @@ - CREATE - DROP - ALTER + - ALTER_SESSION - REPLACE - SET + - SET_SESSION - USE_DB - EXPLAIN - SHOW @@ -127,6 +133,8 @@ - SORT - DESC - TRUNCATE + - CALL + - COMMENT_ON - OTHERS - id: 5 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.name} @@ -183,7 +191,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -201,7 +209,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -225,7 +233,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -249,7 +257,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DML - label: SUB_TYPE @@ -267,7 +275,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DML - label: SUB_TYPE @@ -311,7 +319,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -330,19 +338,19 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 200 - id: 14 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -355,19 +363,19 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 10 - id: 15 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -380,19 +388,19 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 10 - id: 16 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -409,12 +417,12 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 100 - id: 17 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.name} @@ -425,7 +433,7 @@ - label: SUB_TYPE value: DDL - label: SUB_TYPE - value: TABLE + value: TABLE - label: SUPPORTED_DIALECT_TYPE value: OB_MYSQL - label: SUPPORTED_DIALECT_TYPE @@ -437,7 +445,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -450,19 +458,19 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 100 - id: 19 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -477,19 +485,19 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 1000 - id: 20 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -509,7 +517,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -527,7 +535,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -545,7 +553,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -560,7 +568,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list.description} type: STRING_LIST @@ -570,7 +578,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -583,7 +591,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets.description} type: STRING_LIST @@ -593,7 +601,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -606,7 +614,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations.description} type: STRING_LIST @@ -616,7 +624,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -631,7 +639,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes.description} type: STRING_LIST @@ -648,7 +656,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -664,7 +672,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -679,19 +687,19 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 100 - id: 29 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -708,7 +716,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern.description} type: STRING @@ -718,7 +726,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -735,7 +743,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern.description} type: STRING @@ -745,7 +753,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -765,7 +773,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -783,7 +791,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -803,7 +811,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -823,7 +831,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -838,7 +846,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list.description} type: STRING_LIST @@ -848,7 +856,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -863,7 +871,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list.description} type: STRING_LIST @@ -873,7 +881,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -893,7 +901,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -908,7 +916,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list.description} type: STRING_LIST @@ -918,7 +926,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -927,31 +935,31 @@ value: ALTER - label: SUPPORTED_DIALECT_TYPE value: OB_ORACLE - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase.description} type: BOOLEAN componentType: RADIO - defaultValues: + defaultValues: - false candidates: - true - - false + - false - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase.description} type: BOOLEAN componentType: RADIO - defaultValues: + defaultValues: - false candidates: - true - - false + - false - id: 40 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -960,12 +968,12 @@ value: ALTER - label: SUPPORTED_DIALECT_TYPE value: OB_ORACLE - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase.description} type: BOOLEAN componentType: RADIO - defaultValues: + defaultValues: - false candidates: - true @@ -974,7 +982,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase.description} type: BOOLEAN componentType: RADIO - defaultValues: + defaultValues: - false candidates: - true @@ -984,7 +992,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -995,19 +1003,19 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 1 - id: 42 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -1025,7 +1033,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1038,7 +1046,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names.description} type: STRING_LIST @@ -1048,7 +1056,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1066,7 +1074,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1079,19 +1087,19 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 10 - id: 46 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1111,7 +1119,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1126,7 +1134,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern.description} type: STRING @@ -1136,7 +1144,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1151,7 +1159,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names.description} type: STRING_LIST @@ -1178,12 +1186,12 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes.description} type: STRING_LIST componentType: SELECT_TAGS - defaultValues: + defaultValues: - int - varchar2 - number diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SqlConsoleInterceptor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SqlConsoleInterceptor.java index 35de0eff18..ec0bbc5ebe 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SqlConsoleInterceptor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SqlConsoleInterceptor.java @@ -170,7 +170,7 @@ private boolean handle(@NonNull SqlAsyncExecuteReq request, @NonNull SqlAsyncExe if (Objects.nonNull(parseResult.getSyntaxError()) && parseResult.getSyntaxError()) { continue; } - if (!allowSqlTypesOpt.get().contains(parseResult.getSqlType().name())) { + if (isViolatedRule(allowSqlTypesOpt.get(), parseResult)) { ruleService.getByRulesetIdAndName(ruleSetId, SqlConsoleRules.ALLOW_SQL_TYPES.getRuleName()) .ifPresent(rule -> { Rule violationRule = new Rule(); @@ -247,6 +247,15 @@ private BasicResult determineSqlType(@NonNull SqlTuple sqlTuple, @NonNull Dialec return basicResult; } + private boolean isViolatedRule(@NonNull List allowSqlTypesOpt, @NonNull BasicResult parseResult) { + for (SqlType sqlType = parseResult.getSqlType(); sqlType != null; sqlType = sqlType.getParentType()) { + if (allowSqlTypesOpt.contains(sqlType.name())) { + return false; + } + } + return true; + } + @Override public int getOrder() { return 2; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java index ad34e2758b..d0a979a73a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java @@ -216,7 +216,8 @@ private boolean checkContainsExternalTable(@NonNull ConnectionSession connection DialectType dialectType = connectionSession.getDialectType(); if (dialectType == DialectType.OB_MYSQL || dialectType == DialectType.OB_ORACLE) { String obVersion = ConnectionSessionUtil.getVersion(connectionSession); - if (VersionUtils.isGreaterThanOrEqualsTo(obVersion, MIN_OB_VERSION_FOR_EXTERNAL_TABLE)) { + if (VersionUtils.isGreaterThanOrEqualsTo(obVersion, MIN_OB_VERSION_FOR_EXTERNAL_TABLE) + && resultSetMetaData != null) { List columnList = resultSetMetaData.getFieldMetaDataList(); Map schemaAndTable2Column = columnList.stream() .collect(Collectors.groupingBy(jcmd -> jcmd.getCatalogName() + "." + jcmd.getTableName(), From de669fad692b3997a030fb4673dfd6092c002e0b Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Thu, 14 Nov 2024 15:26:53 +0800 Subject: [PATCH 029/118] feat(external table): external table support sensitive column (#3821) --- .../dbbrowser/schema/DBSchemaAccessor.java | 10 + .../schema/DBSchemaAccessorSqlMappers.java | 3 +- .../dbbrowser/schema/constant/Statements.java | 2 + .../schema/constant/StatementsFiles.java | 4 +- .../schema/doris/DorisSchemaAccessor.java | 10 + .../MySQLNoLessThan5700SchemaAccessor.java | 10 + ...OBMySQLBetween400And432SchemaAccessor.java | 13 + .../schema/mysql/OBMySQLSchemaAccessor.java | 17 +- ...BOracleBetween410And432SchemaAccessor.java | 16 + .../schema/oracle/OBOracleSchemaAccessor.java | 17 +- .../schema/oracle/OracleSchemaAccessor.java | 10 + .../postgre/PostgresSchemaAccessor.java | 10 + ...bmysql_4_0_x.yaml => obmysql_4_3_2_x.yaml} | 35 ++ .../schema/sql/oboracle/oboracle_4_3_2_x.yaml | 324 ++++++++++++++++++ .../datasecurity/SensitiveColumnService.java | 8 + .../model/DatabaseWithAllColumns.java | 1 + 16 files changed, 486 insertions(+), 4 deletions(-) rename libs/db-browser/src/main/resources/schema/sql/obmysql/{obmysql_4_0_x.yaml => obmysql_4_3_2_x.yaml} (86%) create mode 100644 libs/db-browser/src/main/resources/schema/sql/oboracle/oboracle_4_3_2_x.yaml diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java index 16a6a2ad27..eb9bac6ab5 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java @@ -224,6 +224,16 @@ default List showExternalTables(String schemaName) { */ List listBasicViewColumns(String schemaName, String viewName); + /** + * Get all external table columns(hold only basic info) in the specified schema + */ + Map> listBasicExternalTableColumns(String schemaName); + + /** + * Get all external table columns(hold only basic info) in the specified schema and view + */ + List listBasicExternalTableColumns(String schemaName, String externalTableName); + /** * Get all table and view columns info (hold only basic info: schema, table and column name) in the * specified schema diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorSqlMappers.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorSqlMappers.java index f2c9dc8473..eab8b54a3f 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorSqlMappers.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorSqlMappers.java @@ -46,7 +46,7 @@ public class DBSchemaAccessorSqlMappers { static { SQL_MAPPER_FILE_PATHS.addAll(Arrays.asList( - StatementsFiles.OBMYSQL_40X, + StatementsFiles.OBMYSQL_432x, StatementsFiles.OBMYSQL_3X, StatementsFiles.OBMYSQL_2276, StatementsFiles.OBMYSQL_225X, @@ -56,6 +56,7 @@ public class DBSchemaAccessorSqlMappers { StatementsFiles.OBORACLE_3_x, StatementsFiles.OBORACLE_4_0_x, StatementsFiles.OBORACLE_4_1_x, + StatementsFiles.OBORACLE_4_3_2_x, StatementsFiles.ORACLE_11_g)); for (String path : SQL_MAPPER_FILE_PATHS) { URL url = DBSchemaAccessorSqlMappers.class.getClassLoader().getResource(path); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/Statements.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/Statements.java index edacfc2a80..61521e4982 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/Statements.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/Statements.java @@ -21,6 +21,8 @@ * @Description: [] */ public final class Statements { + public static final String LIST_BASIC_EXTERNAL_TABLE_COLUMNS = "list-basic-external-table-columns"; + public static final String LIST_BASIC_SCHEMA_EXTERNAL_TABLE_COLUMNS = "list-basic-schema-external-table-columns"; public static final String LIST_BASIC_TABLE_COLUMNS = "list-basic-table-columns"; public static final String LIST_BASIC_SCHEMA_TABLE_COLUMNS = "list-basic-schema-table-columns"; public static final String LIST_BASIC_VIEW_COLUMNS = "list-basic-view-columns"; diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/StatementsFiles.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/StatementsFiles.java index b85ca26295..d5b2c63895 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/StatementsFiles.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/StatementsFiles.java @@ -21,7 +21,7 @@ * @Description: [] */ public final class StatementsFiles { - public static final String OBMYSQL_40X = "schema/sql/obmysql/obmysql_4_0_x.yaml"; + public static final String OBMYSQL_432x = "schema/sql/obmysql/obmysql_4_3_2_x.yaml"; public static final String OBMYSQL_3X = "schema/sql/obmysql/obmysql_3_x.yaml"; @@ -35,6 +35,8 @@ public final class StatementsFiles { public static final String MYSQL_5_6_x = "schema/sql/mysql/mysql_5_6_x.yaml"; + public static final String OBORACLE_4_3_2_x = "schema/sql/oboracle/oboracle_4_3_2_x.yaml"; + public static final String OBORACLE_4_1_x = "schema/sql/oboracle/oboracle_4_1_x.yaml"; public static final String OBORACLE_4_0_x = "schema/sql/oboracle/oboracle_4_0_x.yaml"; diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java index d2b9bdfaa7..6ac4355a26 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java @@ -472,6 +472,16 @@ public List listBasicViewColumns(String schemaName, String viewNa return jdbcOperations.query(sql, new Object[] {schemaName, viewName}, listBasicTableColumnRowMapper()); } + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + @Override public Map> listBasicColumnsInfo(String schemaName) { String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_COLUMNS_INFO); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java index d5a87e7e85..bdeb69f07b 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java @@ -493,6 +493,16 @@ public List listBasicViewColumns(String schemaName, String viewNa return jdbcOperations.query(sql, new Object[] {schemaName, viewName}, listBasicTableColumnRowMapper()); } + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("not support yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("not support yet"); + } + @Override public Map> listBasicColumnsInfo(String schemaName) { String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_COLUMNS_INFO); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween400And432SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween400And432SchemaAccessor.java index 477c273e58..650969ece9 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween400And432SchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween400And432SchemaAccessor.java @@ -16,10 +16,12 @@ package com.oceanbase.tools.dbbrowser.schema.mysql; import java.util.List; +import java.util.Map; import org.springframework.jdbc.core.JdbcOperations; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBTableColumn; /** * @description: applicable to OB [4.0.0,4.3.2) @@ -56,4 +58,15 @@ public List listExternalTables(String schemaName, String table public boolean isExternalTable(String schemaName, String tableName) { return false; } + + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("not support yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("not support yet"); + } + } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java index e46d42073a..a9b9f9a4e1 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java @@ -42,6 +42,7 @@ import com.oceanbase.tools.dbbrowser.parser.SqlParser; import com.oceanbase.tools.dbbrowser.parser.result.ParseSqlResult; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessorSqlMappers; +import com.oceanbase.tools.dbbrowser.schema.constant.Statements; import com.oceanbase.tools.dbbrowser.schema.constant.StatementsFiles; import com.oceanbase.tools.dbbrowser.util.DBSchemaAccessorUtil; import com.oceanbase.tools.dbbrowser.util.MySQLSqlBuilder; @@ -77,7 +78,7 @@ public List showDatabases() { public OBMySQLSchemaAccessor(JdbcOperations jdbcOperations) { super(jdbcOperations); - this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBMYSQL_40X); + this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBMYSQL_432x); } @Override @@ -432,6 +433,20 @@ public boolean syncExternalTableFiles(String schemaName, String tableName) { return true; } + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_EXTERNAL_TABLE_COLUMNS); + List tableColumns = jdbcOperations.query(sql, new Object[] {schemaName, schemaName}, + listBasicTableColumnRowMapper()); + return tableColumns.stream().collect(Collectors.groupingBy(DBTableColumn::getTableName)); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + String sql = sqlMapper.getSql(Statements.LIST_BASIC_EXTERNAL_TABLE_COLUMNS); + return jdbcOperations.query(sql, new Object[] {schemaName, externalTableName}, listBasicTableColumnRowMapper()); + } + @Override protected void correctColumnPrecisionIfNeed(List tableColumns) {} } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java index 742232d444..c5467643d5 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java @@ -16,11 +16,15 @@ package com.oceanbase.tools.dbbrowser.schema.oracle; import java.util.List; +import java.util.Map; import org.springframework.jdbc.core.BeanPropertyRowMapper; import org.springframework.jdbc.core.JdbcOperations; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBTableColumn; +import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessorSqlMappers; +import com.oceanbase.tools.dbbrowser.schema.constant.StatementsFiles; import com.oceanbase.tools.dbbrowser.util.OracleDataDictTableNames; import com.oceanbase.tools.dbbrowser.util.OracleSqlBuilder; import com.oceanbase.tools.dbbrowser.util.StringUtils; @@ -36,6 +40,7 @@ public class OBOracleBetween410And432SchemaAccessor extends OBOracleSchemaAccess public OBOracleBetween410And432SchemaAccessor(JdbcOperations jdbcOperations, OracleDataDictTableNames dataDictTableNames) { super(jdbcOperations, dataDictTableNames); + this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBORACLE_4_1_x); } @Override @@ -97,4 +102,15 @@ public List listTables(String schemaName, String tableNameLike return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); } + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("not support yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("not support yet"); + } + + } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java index e0dd96a7d8..4ca56d6f6e 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java @@ -71,6 +71,7 @@ import com.oceanbase.tools.dbbrowser.parser.result.ParseOraclePLResult; import com.oceanbase.tools.dbbrowser.parser.result.ParseSqlResult; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessorSqlMappers; +import com.oceanbase.tools.dbbrowser.schema.constant.Statements; import com.oceanbase.tools.dbbrowser.schema.constant.StatementsFiles; import com.oceanbase.tools.dbbrowser.util.DBSchemaAccessorUtil; import com.oceanbase.tools.dbbrowser.util.OracleDataDictTableNames; @@ -103,7 +104,7 @@ public class OBOracleSchemaAccessor extends OracleSchemaAccessor { public OBOracleSchemaAccessor(JdbcOperations jdbcOperations, OracleDataDictTableNames dataDictTableNames) { super(jdbcOperations, dataDictTableNames); - this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBORACLE_4_1_x); + this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBORACLE_4_3_2_x); } @Override @@ -1104,6 +1105,20 @@ public List listTables(String schemaName, String tableNameLike return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); } + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_EXTERNAL_TABLE_COLUMNS); + List tableColumns = + jdbcOperations.query(sql, new Object[] {schemaName, schemaName}, listBasicColumnsRowMapper()); + return tableColumns.stream().collect(Collectors.groupingBy(DBTableColumn::getTableName)); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + String sql = sqlMapper.getSql(Statements.LIST_BASIC_EXTERNAL_TABLE_COLUMNS); + return jdbcOperations.query(sql, new Object[] {schemaName, externalTableName}, listBasicColumnsRowMapper()); + } + private List commonShowTablesLike(String schemaName, String tableNameLike, @NonNull DBObjectType tableType) { OracleSqlBuilder sb = new OracleSqlBuilder(); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java index 0cb247c93d..9591d11fea 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java @@ -644,6 +644,16 @@ public List listBasicViewColumns(String schemaName, String viewNa return jdbcOperations.query(sql, new Object[] {schemaName, viewName}, listBasicColumnsRowMapper()); } + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("not support yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("not support yet"); + } + @Override public Map> listBasicColumnsInfo(String schemaName) { String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_COLUMNS_INFO); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java index 637d26a651..c76721fd57 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java @@ -281,6 +281,16 @@ public List listBasicViewColumns(String schemaName, String viewNa throw new UnsupportedOperationException("Not supported yet"); } + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("not support yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("not support yet"); + } + @Override public Map> listBasicColumnsInfo(String schemaName) { throw new UnsupportedOperationException("Not supported yet"); diff --git a/libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_0_x.yaml b/libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_3_2_x.yaml similarity index 86% rename from libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_0_x.yaml rename to libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_3_2_x.yaml index bb8fe23387..601d4e6d92 100644 --- a/libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_0_x.yaml +++ b/libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_3_2_x.yaml @@ -34,6 +34,41 @@ sqls: ORDER BY TABLE_NAME ASC, ORDINAL_POSITION ASC + list-basic-external-table-columns: |- + SELECT + TABLE_SCHEMA, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COLUMN_COMMENT + FROM + information_schema.columns + WHERE + TABLE_SCHEMA = ? AND TABLE_NAME = ? + ORDER BY + ORDINAL_POSITION ASC + list-basic-schema-external-table-columns: |- + SELECT + TABLE_SCHEMA, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COLUMN_COMMENT + FROM + information_schema.columns + WHERE + TABLE_SCHEMA = ? + AND TABLE_NAME IN ( + SELECT + TABLE_NAME + FROM + information_schema.tables + WHERE + TABLE_SCHEMA = ? AND TABLE_TYPE = 'EXTERNAL TABLE' + ) + ORDER BY + TABLE_NAME ASC, + ORDINAL_POSITION ASC list-basic-view-columns: |- SELECT TABLE_SCHEMA, diff --git a/libs/db-browser/src/main/resources/schema/sql/oboracle/oboracle_4_3_2_x.yaml b/libs/db-browser/src/main/resources/schema/sql/oboracle/oboracle_4_3_2_x.yaml new file mode 100644 index 0000000000..98dfe4076a --- /dev/null +++ b/libs/db-browser/src/main/resources/schema/sql/oboracle/oboracle_4_3_2_x.yaml @@ -0,0 +1,324 @@ +sqls: + list-basic-table-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? AND TABLE_NAME = ? AND USER_GENERATED='YES' + ORDER BY + COLUMN_ID ASC + list-basic-schema-table-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? + AND TABLE_NAME IN ( + SELECT + TABLE_NAME + FROM + SYS.ALL_TABLES + WHERE + OWNER = ? AND EXTERNAL='NO' + ) AND USER_GENERATED='YES' + ORDER BY + TABLE_NAME ASC, + COLUMN_ID ASC + list-basic-external-table-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? AND TABLE_NAME = ? AND USER_GENERATED='YES' + ORDER BY + COLUMN_ID ASC + list-basic-schema-external-table-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? + AND TABLE_NAME IN ( + SELECT + TABLE_NAME + FROM + SYS.ALL_TABLES + WHERE + OWNER = ? AND EXTERNAL='YES' + ) AND USER_GENERATED='YES' + ORDER BY + TABLE_NAME ASC, + COLUMN_ID ASC + list-basic-view-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? AND TABLE_NAME = ? AND USER_GENERATED='YES' + ORDER BY + COLUMN_ID ASC + list-basic-schema-view-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? + AND TABLE_NAME IN ( + SELECT + VIEW_NAME + FROM + SYS.ALL_VIEWS + WHERE + OWNER = ? + ) AND USER_GENERATED='YES' + ORDER BY + TABLE_NAME ASC, + COLUMN_ID ASC + list-basic-schema-columns-info: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME + FROM + SYS.ALL_TAB_COLS + WHERE + OWNER = ? + ORDER BY + TABLE_NAME ASC, + COLUMN_ID ASC + list-table-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_ID, + COLUMN_NAME, + DATA_TYPE, + DATA_SCALE, + DATA_PRECISION, + DATA_LENGTH, + CHAR_LENGTH, + DATA_TYPE_MOD, + CHAR_USED, + NULLABLE, + DATA_DEFAULT, + HIDDEN_COLUMN, + VIRTUAL_COLUMN + FROM + SYS.ALL_TAB_COLS + WHERE + OWNER = ? AND TABLE_NAME = ? and USER_GENERATED='YES' + ORDER BY + COLUMN_ID ASC + list-schema-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_ID, + COLUMN_NAME, + DATA_TYPE, + DATA_SCALE, + DATA_PRECISION, + DATA_LENGTH, + CHAR_LENGTH, + DATA_TYPE_MOD, + CHAR_USED, + NULLABLE, + DATA_DEFAULT, + HIDDEN_COLUMN, + VIRTUAL_COLUMN + FROM + SYS.ALL_TAB_COLS + WHERE + OWNER = ? and USER_GENERATED='YES' + ORDER BY + COLUMN_ID ASC + list-table-indexes: |- + SELECT + IDX.OWNER, + IDX.TABLE_OWNER, + IDX.TABLE_NAME, + IDX.INDEX_NAME, + IDX_COL.COLUMN_NAME, + IDX_COL.COLUMN_POSITION, + IDX.INDEX_TYPE, + IDX.UNIQUENESS, + IDX.COMPRESSION, + IDX.VISIBILITY, + IDX.STATUS + FROM + SYS.ALL_IND_COLUMNS IDX_COL + LEFT JOIN + SYS.ALL_INDEXES IDX ON IDX_COL.TABLE_OWNER = IDX.TABLE_OWNER + AND IDX_COL.TABLE_NAME = IDX.TABLE_NAME + AND IDX_COL.INDEX_NAME = IDX.INDEX_NAME + WHERE + IDX.TABLE_OWNER = ? AND IDX.TABLE_NAME = ? + ORDER BY + IDX.INDEX_NAME, + IDX_COL.COLUMN_POSITION ASC + list-schema-index: |- + SELECT + IDX.OWNER, + IDX.TABLE_OWNER, + IDX.TABLE_NAME, + IDX.INDEX_NAME, + IDX_COL.COLUMN_NAME, + IDX_COL.COLUMN_POSITION, + IDX.INDEX_TYPE, + IDX.UNIQUENESS, + IDX.COMPRESSION, + IDX.VISIBILITY, + IDX.STATUS + FROM + SYS.ALL_IND_COLUMNS IDX_COL + LEFT JOIN + SYS.ALL_INDEXES IDX ON IDX_COL.TABLE_OWNER = IDX.TABLE_OWNER + AND IDX_COL.TABLE_NAME = IDX.TABLE_NAME + AND IDX_COL.INDEX_NAME = IDX.INDEX_NAME + WHERE + IDX.TABLE_OWNER = ? + ORDER BY + IDX.TABLE_NAME, + IDX.INDEX_NAME, + IDX_COL.COLUMN_POSITION ASC + list-table-constraints: |- + SELECT + t1.OWNER, + t1.CONSTRAINT_NAME, + t1.CONSTRAINT_TYPE, + t1.TABLE_NAME, + t1.SEARCH_CONDITION, + t1.R_OWNER, + t1.R_CONSTRAINT_NAME, + t1.DELETE_RULE, + t1.STATUS, + t1.DEFERRABLE, + t1.DEFERRED, + t1.VALIDATED, + t2.TABLE_NAME as R_TABLE_NAME, + t2.COLUMN_NAME as R_COLUMN_NAME, + t3.position as POSITION, + t3.COLUMN_NAME as COLUMN_NAME + FROM + SYS.ALL_CONSTRAINTS t1 + JOIN SYS.ALL_CONS_COLUMNS t3 on t1.CONSTRAINT_NAME = t3.CONSTRAINT_NAME and t1.OWNER = t3.OWNER + LEFT JOIN SYS.ALL_CONS_COLUMNS t2 on t2.CONSTRAINT_NAME = t1.R_CONSTRAINT_NAME + AND t2.OWNER = t1.R_OWNER + WHERE + t1.OWNER = ? and t1.TABLE_NAME = ? + ORDER BY + t1.CONSTRAINT_NAME, + t3.position ASC + list-schema-constraints: |- + SELECT + t1.OWNER, + t1.CONSTRAINT_NAME, + t1.CONSTRAINT_TYPE, + t1.TABLE_NAME, + t1.SEARCH_CONDITION, + t1.R_OWNER, + t1.R_CONSTRAINT_NAME, + t1.DELETE_RULE, + t1.STATUS, + t1.DEFERRABLE, + t1.DEFERRED, + t1.VALIDATED, + t2.TABLE_NAME as R_TABLE_NAME, + t2.COLUMN_NAME as R_COLUMN_NAME, + t3.position as POSITION, + t3.COLUMN_NAME as COLUMN_NAME + FROM + SYS.ALL_CONSTRAINTS t1 + JOIN SYS.ALL_CONS_COLUMNS t3 on t1.CONSTRAINT_NAME = t3.CONSTRAINT_NAME and t1.OWNER = t3.OWNER + LEFT JOIN SYS.ALL_CONS_COLUMNS t2 on t2.CONSTRAINT_NAME = t1.R_CONSTRAINT_NAME + AND t2.OWNER = t1.R_OWNER + WHERE + t1.OWNER = ? + ORDER BY + t1.TABLE_NAME, + t1.CONSTRAINT_NAME, + t3.position ASC + get-partition-option: |- + SELECT + PARTITIONING_TYPE + FROM + SYS.ALL_PART_TABLES + WHERE + OWNER = ? + AND TABLE_NAME = ? + list-partitions-options: |- + SELECT + TABLE_NAME, + PARTITIONING_TYPE + FROM + SYS.ALL_PART_TABLES + WHERE + OWNER = ? + list-partition-definitions: |- + SELECT + PARTITION_NAME, + PARTITION_POSITION, + HIGH_VALUE + FROM + SYS.ALL_TAB_PARTITIONS + WHERE + TABLE_OWNER = ? + AND TABLE_NAME = ? + ORDER BY + PARTITION_POSITION ASC + list-partitions-definitions: |- + SELECT + TABLE_NAME, + PARTITION_NAME, + PARTITION_POSITION, + HIGH_VALUE + FROM + SYS.ALL_TAB_PARTITIONS + WHERE + TABLE_OWNER = ? + ORDER BY + PARTITION_POSITION ASC + list-database: |- + select + USERNAME, + USERID + from + ALL_USERS + get-database: |- + SELECT + USERNAME, + USERID + from + SYS.ALL_USERS + WHERE + USERNAME = ? \ No newline at end of file diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java index 1bc736c949..45d737a979 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java @@ -46,7 +46,9 @@ import com.oceanbase.odc.core.authority.util.PreAuthenticate; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.NotFoundException; @@ -145,6 +147,12 @@ public List listColumns(@NotNull Long projectId, @NotEmp accessor.listBasicTableColumns(database.getName()), exists)); databaseColumn.setView2Columns(getFilteringExistColumns(database.getId(), accessor.listBasicViewColumns(database.getName()), exists)); + DialectType dialectType = session.getDialectType(); + String version = ConnectionSessionUtil.getVersion(session); + if (versionDiffConfigService.isExternalTableSupported(dialectType, version)) { + databaseColumn.setExternalTable2Columns(getFilteringExistColumns(database.getId(), + accessor.listBasicExternalTableColumns(database.getName()), exists)); + } databaseColumn.setDataTypeUnits(versionDiffConfigService.getDatatypeList(session)); if (!databaseColumn.getTable2Columns().isEmpty() || !databaseColumn.getView2Columns().isEmpty()) { databaseColumns.add(databaseColumn); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/model/DatabaseWithAllColumns.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/model/DatabaseWithAllColumns.java index 2cbd7e8444..580c193c39 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/model/DatabaseWithAllColumns.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/model/DatabaseWithAllColumns.java @@ -35,6 +35,7 @@ public class DatabaseWithAllColumns { private String databaseName; private Map> table2Columns; private Map> view2Columns; + private Map> externalTable2Columns; /** * Mapping from database type to show type, used for displaying column type icon */ From dd8a4ad2b5b9c6f81d1dd4dd43a5bf0a7895dfdf Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Mon, 18 Nov 2024 10:03:47 +0800 Subject: [PATCH 030/118] enhancement(external table):add swagger annotation for sync external table files (#3840) --- .../odc/server/web/controller/v2/DBTableController.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java index 2cf205b45d..e2118bd231 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java @@ -45,6 +45,8 @@ import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.model.datatype.DataType; +import io.swagger.annotations.ApiOperation; + @RestController @RequestMapping("api/v2/connect/sessions") public class DBTableController { @@ -122,6 +124,7 @@ public ListResponse getPartitionKeyDataTypes(@PathVariable String sess return Responses.list(this.partitionPlanService.getPartitionKeyDataTypes(sessionId, databaseId, tableName)); } + @ApiOperation(value = "syncExternalTableFiles", notes = "sync external table files") @PostMapping(value = "/{sessionId}/databases/{databaseName}/externalTables/" + "{externalTableName}/syncExternalTableFiles") public SuccessResponse syncExternalTableFiles(@PathVariable String sessionId, From 256779b4e8887b9d886965fa636467a30e327c40 Mon Sep 17 00:00:00 2001 From: yizhouxw Date: Wed, 20 Nov 2024 12:09:25 +0800 Subject: [PATCH 031/118] build: recover submodule --- .gitmodules | 4 ++++ client | 1 + 2 files changed, 5 insertions(+) create mode 160000 client diff --git a/.gitmodules b/.gitmodules index e44ed4440a..f52bde7184 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,7 @@ +[submodule "client"] + path = client + url = https://github.com/oceanbase/odc-client.git + branch = dev-4.3.3 [submodule "build-resource"] path = build-resource url = https://github.com/oceanbase/odc-build-resource.git diff --git a/client b/client new file mode 160000 index 0000000000..0021be9fee --- /dev/null +++ b/client @@ -0,0 +1 @@ +Subproject commit 0021be9feed6d121c190f1aa9565c0ab9fa34dc5 From 3fa8ec132874e61a4f55153e33b9ef5edc074d27 Mon Sep 17 00:00:00 2001 From: LioRoger Date: Wed, 20 Nov 2024 17:07:21 +0800 Subject: [PATCH 032/118] feat(logicaldatabase): add missed code for data.sql when merge main to dev/4.3.x (#3882) --- server/odc-server/src/main/resources/data.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/odc-server/src/main/resources/data.sql b/server/odc-server/src/main/resources/data.sql index c2f78f4bc9..010deb027f 100644 --- a/server/odc-server/src/main/resources/data.sql +++ b/server/odc-server/src/main/resources/data.sql @@ -852,3 +852,6 @@ INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('o -- INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('server.servlet.session.cookie.secure', 'false', 'Enable secure cookie or not, default value false') ON DUPLICATE KEY UPDATE `id`=`id`; + +INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('odc.features.logicaldatabase.enabled', 'true', + 'Whether to enable the logical database feature, default is true, indicating enabled.') ON DUPLICATE KEY UPDATE `id`=`id`; From c4c5f60e99aae6d192f9c746340a35a9ed32b198 Mon Sep 17 00:00:00 2001 From: kiko Date: Wed, 20 Nov 2024 18:13:58 +0800 Subject: [PATCH 033/118] feat(dlm): data archive support generating dynamic target table name (#3883) * feat:data archive support generating dynamic target table name. * response comment. --- .../com/oceanbase/odc/service/schedule/job/DataArchiveJob.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java index 029c1ddcd8..ef6b448532 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java @@ -54,6 +54,9 @@ private void executeInTaskFramework(JobExecutionContext context) { dataArchiveParameters.getVariables(), context.getFireTime()) : ""); + tableConfig.setTargetTableName(DataArchiveConditionUtil.parseCondition(tableConfig.getTargetTableName(), + dataArchiveParameters.getVariables(), + context.getFireTime())); } parameters.setDeleteAfterMigration(dataArchiveParameters.isDeleteAfterMigration()); parameters.setMigrationInsertAction(dataArchiveParameters.getMigrationInsertAction()); From cf2aaa367854ff02e335b13a106c5525549250a7 Mon Sep 17 00:00:00 2001 From: LioRoger Date: Wed, 20 Nov 2024 21:06:42 +0800 Subject: [PATCH 034/118] builds: merge from main into dev/4.3.x (#3884) * add missed code when merge main to dev/4.3.x * add missed code when merge main to dev/4.3.x * add missed code when merge main to dev/4.3.x --- .../oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java | 4 ++-- .../com/oceanbase/odc/agent/runtime/TaskContainerTest.java | 4 +++- .../oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java index 5f1d32325a..59b6279ab3 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.agent.runtime; import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.common.util.ExceptionUtils; import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.util.JobUtils; @@ -26,7 +27,6 @@ * @since 4.2.4 */ class DefaultTaskResultBuilder { - public static TaskResult build(TaskContainer taskContainer) { TaskResult result = new TaskResult(); Task task = taskContainer.getTask(); @@ -39,6 +39,6 @@ public static TaskResult build(TaskContainer taskContainer) { } public static void assignErrorMessage(TaskResult result, Throwable e) { - result.setErrorMessage(null == e ? null : e.getMessage()); + result.setErrorMessage(null == e ? null : ExceptionUtils.getRootCauseReason(e, 3)); } } diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java index 38f971709d..f978141bf0 100644 --- a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java @@ -25,6 +25,7 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; import com.oceanbase.odc.service.task.Task; @@ -81,7 +82,8 @@ public void testExceptionListenerWithException() { TaskReporter taskReporter = taskContainer.taskMonitor.getReporter(); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskResult.class); Mockito.verify(taskReporter).report(ArgumentMatchers.any(), argumentCaptor.capture()); - Assert.assertEquals(argumentCaptor.getValue().getErrorMessage(), "exception should be thrown"); + Assert.assertTrue( + StringUtils.contains(argumentCaptor.getValue().getErrorMessage(), "exception should be thrown")); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java index a0e286b112..11822ca176 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java @@ -184,6 +184,7 @@ public boolean start() throws Exception { break; } log.warn("Sql task execution failed, will continue to execute next statement.", e); + context.getExceptionListener().onException(e); } } result.setTotalStatements(index); From 47099baee2d37ac2ddc5ee5574316c29b62985cb Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Thu, 21 Nov 2024 11:31:30 +0800 Subject: [PATCH 035/118] feat(regulation): sql check and sql console rules support for Oracle (#3877) * support sql check for oracle mode * fix duplicated metadata --- .../init/regulation-rule-applying.yaml | 200 ++++++++++++++++++ .../init/regulation-rule-metadata.yaml | 100 +++++++++ .../odc/service/sqlcheck/BaseSqlChecker.java | 2 +- .../RestrictColumnNameCaseFactory.java | 2 +- .../factory/RestrictTableNameCaseFactory.java | 2 +- .../sqlcheck/rule/ColumnCharsetExists.java | 2 +- .../sqlcheck/rule/ColumnCollationExists.java | 2 +- .../sqlcheck/rule/ColumnNameInBlackList.java | 2 +- .../rule/ForeignConstraintExists.java | 2 +- .../sqlcheck/rule/NoDefaultValueExists.java | 2 +- .../sqlcheck/rule/NoIndexNameExists.java | 2 +- .../sqlcheck/rule/NoPrimaryKeyExists.java | 2 +- .../sqlcheck/rule/NoPrimaryKeyNameExists.java | 2 +- .../sqlcheck/rule/NoSpecificColumnExists.java | 2 +- .../sqlcheck/rule/NoValidWhereClause.java | 2 +- .../sqlcheck/rule/NoWhereClauseExists.java | 2 +- .../NotNullColumnWithoutDefaultValue.java | 2 +- .../rule/OracleColumnCalculation.java | 4 +- .../sqlcheck/rule/OracleLeftFuzzyMatch.java | 4 +- .../rule/OracleMissingRequiredColumns.java | 4 +- .../rule/OracleNoColumnCommentExists.java | 3 +- .../rule/OracleNoNotNullAtInExpression.java | 4 +- .../rule/OracleNoTableCommentExists.java | 3 +- .../sqlcheck/rule/OracleOfflineDdlExists.java | 3 +- .../rule/OracleRestrictColumnNameCase.java | 3 +- .../rule/OracleRestrictIndexDataTypes.java | 4 +- .../rule/OracleRestrictPKDataTypes.java | 4 +- .../rule/OracleRestrictTableNameCase.java | 3 +- .../rule/OracleTooManyAlterStatement.java | 4 +- .../rule/PreferLocalOutOfLineIndex.java | 2 +- .../rule/ProhibitedDatatypeExists.java | 2 +- .../sqlcheck/rule/RestrictColumnNotNull.java | 2 +- .../rule/RestrictDropObjectTypes.java | 2 +- .../sqlcheck/rule/RestrictIndexNaming.java | 2 +- .../sqlcheck/rule/RestrictPKNaming.java | 2 +- .../rule/RestrictUniqueIndexNaming.java | 2 +- .../sqlcheck/rule/SelectStarExists.java | 2 +- .../sqlcheck/rule/SyntaxErrorExists.java | 2 +- .../sqlcheck/rule/TableNameInBlackList.java | 2 +- .../sqlcheck/rule/TooLongCharLength.java | 2 +- .../rule/TooManyColumnDefinition.java | 2 +- .../rule/TooManyColumnRefInIndex.java | 2 +- .../rule/TooManyColumnRefInPrimaryKey.java | 2 +- .../sqlcheck/rule/TooManyInExpression.java | 2 +- .../sqlcheck/rule/TooManyOutOfLineIndex.java | 2 +- .../sqlcheck/rule/TooManyTableJoin.java | 2 +- 46 files changed, 356 insertions(+), 51 deletions(-) diff --git a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml index a949db4027..9f58372854 100644 --- a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml +++ b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml @@ -6,6 +6,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -16,6 +17,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -26,6 +28,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name}":1000}' @@ -36,6 +39,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name}":["SELECT", "EXPLAIN", "SHOW", "HELP", "START_TRANS", "COMMIT", "ROLLBACK", "SORT", "DESC"]}' @@ -46,6 +50,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name}":1000}' @@ -56,6 +61,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -66,6 +72,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -76,6 +83,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -86,6 +94,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -97,6 +106,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -107,6 +117,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -117,6 +128,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name}":100000}' @@ -127,6 +139,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name}":["UPDATE", "DELETE", "INSERT", "SELECT", "CREATE", "DROP", "ALTER", "REPLACE", "SET", "USE_DB", "EXPLAIN", "SHOW", "HELP", "START_TRANS", "COMMIT", "ROLLBACK", "SORT", "DESC", "TRUNCATE", "OTHERS"]}' @@ -137,6 +150,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name}":1000}' @@ -147,6 +161,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -157,6 +172,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -167,6 +183,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -178,6 +195,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -188,6 +206,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -198,6 +217,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name}":100000}' @@ -208,6 +228,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name}":["UPDATE", "DELETE", "INSERT", "SELECT", "SET", "REPLACE", "EXPLAIN", "SHOW", "HELP", "START_TRANS", "COMMIT", "ROLLBACK", "SORT", "DESC", "TRUNCATE"]}' @@ -218,6 +239,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name}":1000}' @@ -228,6 +250,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -238,6 +261,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -248,6 +272,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -259,6 +284,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -269,6 +295,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -279,6 +306,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name}":1000}' @@ -289,6 +317,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name}":["SELECT", "EXPLAIN", "SHOW", "HELP", "START_TRANS", "COMMIT", "ROLLBACK", "SORT", "DESC"]}' @@ -299,6 +328,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name}":1000}' @@ -309,6 +339,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -319,6 +350,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -329,6 +361,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -340,6 +373,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -350,6 +384,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -360,6 +395,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -370,6 +406,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{}' - enabled: 1 level: 0 @@ -389,6 +426,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -399,6 +437,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -409,6 +448,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -419,6 +459,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -429,6 +470,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -439,6 +481,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count}":200}' @@ -467,6 +510,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count}":10}' @@ -477,6 +521,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count}":10}' @@ -487,6 +532,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count}":100}' @@ -497,6 +543,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' - enabled: 1 @@ -506,6 +553,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count}":100}' @@ -516,6 +564,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length}":1000}' @@ -526,6 +575,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -536,6 +586,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -546,6 +597,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -556,6 +608,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list}":[]}' @@ -584,6 +637,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes}":["int", "varchar2", "varchar", "number", "float", "bigint"]}' @@ -603,6 +657,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count}":100}' @@ -613,6 +668,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern}":null}' @@ -623,6 +679,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern}":null}' @@ -633,6 +690,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -643,6 +701,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -662,6 +721,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -672,6 +732,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -682,6 +743,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list}":[]}' @@ -692,6 +754,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list}":[]}' @@ -702,6 +765,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -712,6 +776,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list}":[]}' @@ -721,6 +786,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -728,6 +794,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -745,6 +812,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -755,6 +823,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names}":[]}' @@ -774,6 +843,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count}":10}' @@ -784,6 +854,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -794,6 +865,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern}":null}' @@ -804,6 +876,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names}":[]}' @@ -814,6 +887,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes}":["int", "varchar2", "number", "float", "bigint"]}' @@ -833,6 +907,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types}":[]}' @@ -843,6 +918,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -853,6 +929,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -863,6 +940,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -873,6 +951,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{}' - enabled: 1 level: 0 @@ -892,6 +971,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -902,6 +982,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -912,6 +993,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -922,6 +1004,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -932,6 +1015,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -942,6 +1026,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count}":200}' @@ -970,6 +1055,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count}":10}' @@ -980,6 +1066,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count}":10}' @@ -990,6 +1077,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count}":100}' @@ -1000,6 +1088,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' - enabled: 1 @@ -1009,6 +1098,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count}":100}' @@ -1019,6 +1109,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length}":1000}' @@ -1029,6 +1120,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1039,6 +1131,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1049,6 +1142,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1059,6 +1153,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list}":[]}' @@ -1087,6 +1182,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes}":["int", "varchar2", "varchar", "number", "float", "bigint"]}' @@ -1106,6 +1202,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count}":100}' @@ -1116,6 +1213,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern}":null}' @@ -1126,6 +1224,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern}":null}' @@ -1136,6 +1235,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1146,6 +1246,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1165,6 +1266,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1175,6 +1277,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1185,6 +1288,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list}":[]}' @@ -1195,6 +1299,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list}":[]}' @@ -1205,6 +1310,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1215,6 +1321,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list}":[]}' @@ -1224,6 +1331,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -1231,6 +1339,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -1248,6 +1357,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1258,6 +1368,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names}":[]}' @@ -1277,6 +1388,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count}":10}' @@ -1287,6 +1399,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1297,6 +1410,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern}":null}' @@ -1307,6 +1421,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names}":[]}' @@ -1317,6 +1432,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes}":["int", "varchar2", "number", "float", "bigint"]}' @@ -1336,6 +1452,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types}":[]}' @@ -1346,6 +1463,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1356,6 +1474,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1366,6 +1485,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1376,6 +1496,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{}' - enabled: 1 level: 1 @@ -1395,6 +1516,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1405,6 +1527,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1415,6 +1538,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1425,6 +1549,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1435,6 +1560,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1445,6 +1571,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count}":200}' @@ -1473,6 +1600,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count}":10}' @@ -1483,6 +1611,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count}":10}' @@ -1493,6 +1622,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count}":100}' @@ -1503,6 +1633,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' - enabled: 1 @@ -1512,6 +1643,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count}":100}' @@ -1522,6 +1654,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length}":1000}' @@ -1532,6 +1665,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1542,6 +1676,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1552,6 +1687,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1562,6 +1698,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list}":[]}' @@ -1590,6 +1727,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes}":["int", "varchar2", "varchar", "number", "float", "bigint"]}' @@ -1609,6 +1747,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count}":100}' @@ -1619,6 +1758,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern}":null}' @@ -1629,6 +1769,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern}":null}' @@ -1639,6 +1780,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1649,6 +1791,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1668,6 +1811,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1678,6 +1822,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1688,6 +1833,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list}":[]}' @@ -1698,6 +1844,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list}":[]}' @@ -1708,6 +1855,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1718,6 +1866,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list}":[]}' @@ -1727,6 +1876,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -1734,6 +1884,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -1751,6 +1902,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1761,6 +1913,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names}":[]}' @@ -1780,6 +1933,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count}":10}' @@ -1790,6 +1944,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1800,6 +1955,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern}":null}' @@ -1810,6 +1966,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names}":[]}' @@ -1820,6 +1977,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes}":["int", "varchar2", "number", "float", "bigint"]}' @@ -1839,6 +1997,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types}":[]}' @@ -1849,6 +2008,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1859,6 +2019,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1869,6 +2030,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{}' - enabled: 1 level: 1 @@ -1888,6 +2050,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1898,6 +2061,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1908,6 +2072,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1918,6 +2083,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1928,6 +2094,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1938,6 +2105,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count}":200}' @@ -1966,6 +2134,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count}":10}' @@ -1976,6 +2145,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count}":10}' @@ -1986,6 +2156,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count}":100}' @@ -1996,6 +2167,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' - enabled: 1 @@ -2005,6 +2177,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count}":100}' @@ -2015,6 +2188,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length}":1000}' @@ -2025,6 +2199,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2035,6 +2210,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2045,6 +2221,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2055,6 +2232,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list}":[]}' @@ -2083,6 +2261,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes}":["int", "varchar2", "varchar", "number", "float", "bigint"]}' @@ -2102,6 +2281,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count}":100}' @@ -2112,6 +2292,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern}":null}' @@ -2122,6 +2303,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern}":null}' @@ -2132,6 +2314,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2142,6 +2325,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2161,6 +2345,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2171,6 +2356,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2181,6 +2367,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list}":[]}' @@ -2191,6 +2378,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list}":[]}' @@ -2201,6 +2389,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2211,6 +2400,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list}":[]}' @@ -2220,6 +2410,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -2227,6 +2418,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -2244,6 +2436,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2254,6 +2447,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names}":[]}' @@ -2273,6 +2467,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count}":10}' @@ -2283,6 +2478,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2293,6 +2489,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern}":null}' @@ -2303,6 +2500,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names}":[]}' @@ -2313,6 +2511,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes}":["int", "varchar2", "number", "float", "bigint"]}' @@ -2332,6 +2531,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types}":[]}' \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml index 5887c63922..e9d18e970d 100644 --- a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml +++ b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml @@ -16,6 +16,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 2 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.description} @@ -36,6 +38,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 3 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.description} @@ -54,6 +58,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.description} @@ -81,6 +87,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.description} @@ -156,6 +164,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.description} @@ -185,6 +195,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 7 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.name} @@ -204,6 +216,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 8 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.description} @@ -228,6 +242,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 9 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.description} @@ -252,6 +268,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 10 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.description} @@ -270,6 +288,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 11 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.description} @@ -292,6 +312,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 12 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.description} @@ -314,6 +336,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 13 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.description} @@ -338,6 +362,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count.description} @@ -363,6 +389,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count.description} @@ -388,6 +416,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count.description} @@ -417,6 +447,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count.description} @@ -440,6 +472,8 @@ value: OB_ORACLE - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 18 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.description} @@ -458,6 +492,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count.description} @@ -485,6 +521,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length.description} @@ -512,6 +550,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 21 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.description} @@ -530,6 +570,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 22 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.description} @@ -548,6 +590,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 23 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.description} @@ -568,6 +612,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list.description} @@ -639,6 +685,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes.description} @@ -687,6 +735,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count.description} @@ -716,6 +766,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern.description} @@ -743,6 +795,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern.description} @@ -768,6 +822,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 32 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.description} @@ -806,6 +862,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 34 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.description} @@ -826,6 +884,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 35 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.description} @@ -846,6 +906,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list.description} @@ -871,6 +933,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list.description} @@ -896,6 +960,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 38 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.description} @@ -916,6 +982,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list.description} @@ -935,6 +1003,8 @@ value: ALTER - label: SUPPORTED_DIALECT_TYPE value: OB_ORACLE + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase.description} @@ -968,6 +1038,8 @@ value: ALTER - label: SUPPORTED_DIALECT_TYPE value: OB_ORACLE + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase.description} @@ -1028,6 +1100,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 43 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.description} @@ -1046,6 +1120,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names.description} @@ -1087,6 +1163,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count.description} @@ -1114,6 +1192,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 47 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.description} @@ -1134,6 +1214,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern.description} @@ -1159,6 +1241,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names.description} @@ -1186,6 +1270,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes.description} @@ -1219,6 +1305,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 51 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.description} @@ -1239,6 +1327,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 52 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.description} @@ -1259,6 +1349,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.metadata.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.metadata.description} @@ -1282,6 +1374,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types.description} @@ -1337,6 +1431,8 @@ value: OB_ORACLE - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 55 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.description} @@ -1376,6 +1472,8 @@ value: OB_ORACLE - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 57 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.description} @@ -1388,6 +1486,8 @@ value: OB_MYSQL - label: SUPPORTED_DIALECT_TYPE value: OB_ORACLE + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 58 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.description} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/BaseSqlChecker.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/BaseSqlChecker.java index 7729f23a5a..c511590f44 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/BaseSqlChecker.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/BaseSqlChecker.java @@ -59,7 +59,7 @@ public List check(@NonNull String sqlScript) { List sqls = null; if (dialectType.isMysql() || dialectType.isDoris()) { sqls = splitByCommentProcessor(sqlScript); - } else if (dialectType == DialectType.OB_ORACLE) { + } else if (dialectType.isOracle()) { if (DEFAULT_DELIMITER.equals(this.delimiter)) { // 如果用户没有改 delimiter 就用现成的分句逻辑 SqlSplitter sqlSplitter = new SqlSplitter(PlSqlLexer.class, this.delimiter); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/RestrictColumnNameCaseFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/RestrictColumnNameCaseFactory.java index a3c66b85fd..ffcec07344 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/RestrictColumnNameCaseFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/RestrictColumnNameCaseFactory.java @@ -46,7 +46,7 @@ public SqlCheckRule generate(@NonNull DialectType dialectType, Map check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, DialectType.OB_MYSQL); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.ORACLE); } private List builds(String sql, Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCollationExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCollationExists.java index e8654ff338..87fde2bab1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCollationExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCollationExists.java @@ -64,7 +64,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, DialectType.OB_MYSQL, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(String sql, Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnNameInBlackList.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnNameInBlackList.java index 674726d3b1..6060baf29d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnNameInBlackList.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnNameInBlackList.java @@ -79,7 +79,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private String unquoteIdentifier(String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ForeignConstraintExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ForeignConstraintExists.java index de1768a090..089f265fb1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ForeignConstraintExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ForeignConstraintExists.java @@ -77,7 +77,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.OB_MYSQL, DialectType.MYSQL, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List getForeignKeys(Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoDefaultValueExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoDefaultValueExists.java index bb31fef89f..8b095da289 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoDefaultValueExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoDefaultValueExists.java @@ -72,7 +72,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean containsIgnoreCase(String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoIndexNameExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoIndexNameExists.java index e4bb8bb5ca..41ab53ce80 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoIndexNameExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoIndexNameExists.java @@ -91,7 +91,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java index e673fba776..be1071dbfa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java @@ -70,7 +70,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyNameExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyNameExists.java index 5e1ab564b5..1b86ce3c93 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyNameExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyNameExists.java @@ -78,7 +78,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoSpecificColumnExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoSpecificColumnExists.java index 5c6df2ebfd..ba0a510e08 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoSpecificColumnExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoSpecificColumnExists.java @@ -70,7 +70,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List build(String sql, Stream insertTables) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoValidWhereClause.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoValidWhereClause.java index a891fe9752..41762c3eac 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoValidWhereClause.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoValidWhereClause.java @@ -98,7 +98,7 @@ protected List getExpressionIsAlwaysTrueOrFalse(List whe @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.DORIS, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoWhereClauseExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoWhereClauseExists.java index 6dcc1b574e..635db195b7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoWhereClauseExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoWhereClauseExists.java @@ -63,7 +63,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.DORIS, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NotNullColumnWithoutDefaultValue.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NotNullColumnWithoutDefaultValue.java index 689f8473b0..828e48a140 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NotNullColumnWithoutDefaultValue.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NotNullColumnWithoutDefaultValue.java @@ -65,7 +65,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(String sql, Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleColumnCalculation.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleColumnCalculation.java index 76398451ec..75c99cc5d3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleColumnCalculation.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleColumnCalculation.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import com.oceanbase.odc.core.shared.constant.DialectType; @@ -40,7 +40,7 @@ protected boolean containsColumnReference(Expression expr) { @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleLeftFuzzyMatch.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleLeftFuzzyMatch.java index 1dc253e873..2a2fe98fd0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleLeftFuzzyMatch.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleLeftFuzzyMatch.java @@ -16,7 +16,7 @@ package com.oceanbase.odc.service.sqlcheck.rule; import java.util.ArrayList; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import org.apache.commons.lang3.StringUtils; @@ -51,7 +51,7 @@ protected boolean containsLeftFuzzy(Expression expr) { @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleMissingRequiredColumns.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleMissingRequiredColumns.java index fcc62babbf..5a087502b5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleMissingRequiredColumns.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleMissingRequiredColumns.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -44,7 +44,7 @@ protected String unquoteIdentifier(String identifier) { @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoColumnCommentExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoColumnCommentExists.java index 080567c187..a81f54f71a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoColumnCommentExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoColumnCommentExists.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -102,7 +103,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } private String getKey(CreateTable c) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoNotNullAtInExpression.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoNotNullAtInExpression.java index a6954c40eb..97e1fbb7e7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoNotNullAtInExpression.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoNotNullAtInExpression.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -70,7 +70,7 @@ protected boolean containsNotNullForColumnReference(SelectBody selectBody) { @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } private String getColumnName(RelationReference reference) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java index 2ddd188af2..10ef6ca25e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Set; @@ -86,7 +87,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } private String getKey(CreateTable c) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleOfflineDdlExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleOfflineDdlExists.java index 7392b5afd0..01d940d5f4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleOfflineDdlExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleOfflineDdlExists.java @@ -17,6 +17,7 @@ import java.io.StringReader; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -86,7 +87,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } @Override diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictColumnNameCase.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictColumnNameCase.java index d48be225c8..44bb8e78f7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictColumnNameCase.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictColumnNameCase.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -76,7 +77,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } private boolean verify(@NonNull String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictIndexDataTypes.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictIndexDataTypes.java index 81472b58c4..0649401dd1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictIndexDataTypes.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictIndexDataTypes.java @@ -16,7 +16,7 @@ package com.oceanbase.odc.service.sqlcheck.rule; import java.io.StringReader; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -65,7 +65,7 @@ protected CreateTable getTableFromRemote(JdbcOperations jdbcOperations, String s @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictPKDataTypes.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictPKDataTypes.java index 912e4f672f..a70f83b8e7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictPKDataTypes.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictPKDataTypes.java @@ -16,7 +16,7 @@ package com.oceanbase.odc.service.sqlcheck.rule; import java.io.StringReader; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -65,7 +65,7 @@ protected CreateTable getTableFromRemote(JdbcOperations jdbcOperations, String s @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictTableNameCase.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictTableNameCase.java index c590ccbd9b..a2d1a1fd86 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictTableNameCase.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictTableNameCase.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -89,7 +90,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } private boolean verify(@NonNull String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleTooManyAlterStatement.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleTooManyAlterStatement.java index 60b0701149..3477cab524 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleTooManyAlterStatement.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleTooManyAlterStatement.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import com.oceanbase.odc.core.shared.constant.DialectType; @@ -43,7 +43,7 @@ protected String unquoteIdentifier(String identifier) { @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/PreferLocalOutOfLineIndex.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/PreferLocalOutOfLineIndex.java index 804c894b5d..bd6f88735b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/PreferLocalOutOfLineIndex.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/PreferLocalOutOfLineIndex.java @@ -83,7 +83,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ProhibitedDatatypeExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ProhibitedDatatypeExists.java index 8c7a92a42b..50043e5da7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ProhibitedDatatypeExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ProhibitedDatatypeExists.java @@ -72,7 +72,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(String sql, Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictColumnNotNull.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictColumnNotNull.java index b9a0a057e7..b79b1e2542 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictColumnNotNull.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictColumnNotNull.java @@ -73,7 +73,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, DialectType.OB_MYSQL, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean containsIgnoreCase(String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictDropObjectTypes.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictDropObjectTypes.java index 5e29bcae45..15e1801625 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictDropObjectTypes.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictDropObjectTypes.java @@ -93,7 +93,7 @@ action, getType(), new Object[] {getDeleteObjectType(action), allTypes})) @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, - DialectType.OB_MYSQL, DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.OB_MYSQL, DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean notAllow(String objectType) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictIndexNaming.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictIndexNaming.java index 9506a17780..020029a77a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictIndexNaming.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictIndexNaming.java @@ -124,7 +124,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean matches(String tableName, String indexName, List columns) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictPKNaming.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictPKNaming.java index 530a556e94..ea894c5f95 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictPKNaming.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictPKNaming.java @@ -109,7 +109,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, DialectType.OB_MYSQL, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean matches(String tableName, String indexName, List columns) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictUniqueIndexNaming.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictUniqueIndexNaming.java index b9e43f3a33..36239ecc92 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictUniqueIndexNaming.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictUniqueIndexNaming.java @@ -128,7 +128,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean matches(String tableName, String indexName, List columns) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SelectStarExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SelectStarExists.java index c6455080a5..da248144eb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SelectStarExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SelectStarExists.java @@ -90,7 +90,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SyntaxErrorExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SyntaxErrorExists.java index b47f1ef9b7..b9820451dd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SyntaxErrorExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SyntaxErrorExists.java @@ -58,7 +58,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TableNameInBlackList.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TableNameInBlackList.java index 91f020a044..a043137165 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TableNameInBlackList.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TableNameInBlackList.java @@ -92,7 +92,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private String unquoteIdentifier(String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooLongCharLength.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooLongCharLength.java index 7607a20f3e..e068949c69 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooLongCharLength.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooLongCharLength.java @@ -70,7 +70,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.OB_MYSQL, DialectType.MYSQL, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(String sql, Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnDefinition.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnDefinition.java index eee3efc366..f910ceded7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnDefinition.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnDefinition.java @@ -81,7 +81,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInIndex.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInIndex.java index 4e9924cacd..3551000d31 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInIndex.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInIndex.java @@ -115,7 +115,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInPrimaryKey.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInPrimaryKey.java index 88942d0baa..6f392f139a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInPrimaryKey.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInPrimaryKey.java @@ -93,7 +93,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyInExpression.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyInExpression.java index 824b9b35da..cd9c676ff3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyInExpression.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyInExpression.java @@ -67,7 +67,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } protected List getTooManyInExprs(List wheres) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyOutOfLineIndex.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyOutOfLineIndex.java index 40bcf66ecb..0a24e41ade 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyOutOfLineIndex.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyOutOfLineIndex.java @@ -87,7 +87,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyTableJoin.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyTableJoin.java index d8f8925913..b612e82ea9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyTableJoin.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyTableJoin.java @@ -88,7 +88,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } protected List getTooManyJoinRefs(List joins) { From 3bb6a90bf06b0457f662e5e29d423f2c080d8089 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Fri, 22 Nov 2024 14:02:21 +0800 Subject: [PATCH 036/118] fix(sql-check): cannot get oracle affect sql rows (#3892) * fix: adjust OracleAffectedRowsExceedLimit loop start index * Refactor affected rows estimation in SQL check rules --- .../sqlcheck/rule/BaseAffectedRowsExceedLimit.java | 8 ++++++-- .../sqlcheck/rule/OracleAffectedRowsExceedLimit.java | 10 +--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java index 7c2953e1d2..5dcb722c4b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java @@ -99,8 +99,12 @@ public long getOBAffectedRows(String originalSql, JdbcOperations jdbcOperations) */ String explainSql = "EXPLAIN " + originalSql; List queryResults = jdbcOperations.query(explainSql, (rs, rowNum) -> rs.getString("Query Plan")); + return getOBAndOracleAffectRowsFromResult(queryResults); + } + + public long getOBAndOracleAffectRowsFromResult(List queryResults) { long estRowsValue = 0; - for (int rowNum = 3; rowNum < queryResults.size(); rowNum++) { + for (int rowNum = 0; rowNum < queryResults.size(); rowNum++) { String resultRow = queryResults.get(rowNum); estRowsValue = getEstRowsValue(resultRow); if (estRowsValue != 0) { @@ -110,7 +114,7 @@ public long getOBAffectedRows(String originalSql, JdbcOperations jdbcOperations) return estRowsValue; } - public long getEstRowsValue(String singleRow) { + private long getEstRowsValue(String singleRow) { String[] parts = singleRow.split("\\|"); if (parts.length > 5) { String value = parts[4].trim(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleAffectedRowsExceedLimit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleAffectedRowsExceedLimit.java index 86abf9cc0a..ed8fb11b75 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleAffectedRowsExceedLimit.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleAffectedRowsExceedLimit.java @@ -114,15 +114,7 @@ private long getOracleAffectedRows(String originalSql, JdbcOperations jdbcOperat String getPlanSql = "SELECT * FROM TABLE(DBMS_XPLAN.DISPLAY('PLAN_TABLE', '" + ODC_TEMP_EXPLAIN_STATEMENT_ID + "', 'ALL'))"; List queryResults = jdbcOperations.query(getPlanSql, (rs, rowNum) -> rs.getString("PLAN_TABLE_OUTPUT")); - long estRowsValue = 0; - for (int rowNum = 5; rowNum < queryResults.size(); rowNum++) { - String resultRow = queryResults.get(rowNum); - estRowsValue = getEstRowsValue(resultRow); - if (estRowsValue != 0) { - break; - } - } - return estRowsValue; + return getOBAndOracleAffectRowsFromResult(queryResults); } } From 19fb7e2b87e862ade0464fdb65eb24544a99b3a1 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Mon, 25 Nov 2024 11:56:24 +0800 Subject: [PATCH 037/118] security: exclude sshd-common from spring-cloud-context (#3901) --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index cc36555862..e6ada027ba 100644 --- a/pom.xml +++ b/pom.xml @@ -503,6 +503,12 @@ org.springframework.cloud spring-cloud-config-server ${spring-cloud-config-server.version} + + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + + org.springframework.cloud From a2c93d297da06b25babdf610f34c972866a4f1ef Mon Sep 17 00:00:00 2001 From: Yanze Li <37955886+LiYZe@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:36:05 +0800 Subject: [PATCH 038/118] docs: replace idea-run-configuration-start-odc-server-2 with correct image (#3890) * doc: replace idea-run-configuration-start-odc-server-2 with correct image * doc: replace idea-run-configuration-start-odc-server-2 with correct image --- docs/en-US/DEVELOPER_GUIDE.md | 5 +++++ ...a-run-configuration-start-odc-server-2.png | Bin 54913 -> 129648 bytes ...a-run-configuration-start-odc-server-3.png | Bin 0 -> 119529 bytes docs/zh-CN/DEVELOPER_GUIDE.md | 5 +++++ 4 files changed, 10 insertions(+) create mode 100644 docs/en-US/images/idea-run-configuration-start-odc-server-3.png diff --git a/docs/en-US/DEVELOPER_GUIDE.md b/docs/en-US/DEVELOPER_GUIDE.md index dfad2e1808..c4c67059d7 100644 --- a/docs/en-US/DEVELOPER_GUIDE.md +++ b/docs/en-US/DEVELOPER_GUIDE.md @@ -407,6 +407,11 @@ The setup for starting OdcServer is shown below. ![image.png](../en-US/images/idea-run-configuration-start-odc-server-2.png) +The setup for environment variables is shown below. + +![image.png](../en-US/images/idea-run-configuration-start-odc-server-3.png) + + # 4. Frontend-backend integration ## 4.1 Front-end and back-end integration based on a static resource server diff --git a/docs/en-US/images/idea-run-configuration-start-odc-server-2.png b/docs/en-US/images/idea-run-configuration-start-odc-server-2.png index 45371ba394a6b79a7f6983f2e05ecfc3fa624f8c..2204690cff7220b17ba71f9410028f2c8a473960 100644 GIT binary patch literal 129648 zcmdSA1zQ|VvoMUiLvUGOafjgU?t$R0!7aEGg1fs02@u@fB|(C_F7EEUuY-#8kk*AYX6rB{-PZnkC5eU@$Nw0ZUO) zC23JnfRdw~nWeQU7?@OO;(J&P)gkOGZ57g5FbEY9y8yogFclFC@B*AJ2`Pvm3}ZmZ z+=MvvEF*Sh?pLxfA~aqjbM!-RWieq*9K??xd4%2OZ(erocE^u5*=#S|PG+`vU{i_? zeH(=DApLZd+kqTFKNb$|Pr~~!CI)cM;6xhs?(RnF&dyMNrj<{g?yi`C>g%*Xxr6Md zmpY4cXm>ua7>kg}TlZF-5L9r#_vgKFUSJ`tD_$SkHG&PH7cg)Ey@;kLFVEafR-cC+b7o-ZG=3o{=NR#&1 zCVj?-Zqv;cVknc-S%Tg;#o{>TFe_&CWM2Dz+4N0bBHYVCrPEMjqsXnQ=I&`-9?c?T z>?uXN3HwQw510wK4UA!G_>5ae(JZ+KM$djR9;-Z z_~lOxOog#-#sj{HSSthMB7!uKjRsl|8|@2!?B>aR+{%d*!>0>hh1k&Wrv?h;M!zYL z4KPl;E^KlSyy)XFN8pZ?@d6043A~FXDN7wl96;R(sDl?P3lb(0UYQH&YC47X7Y=JO zUWQN>$xzt|Lup#QM{P57XYG6%mG0ZTBFTj1Nrq-A$Z@ah>JS z+iKL35qsf;>wnhhn+7i!@kHDrukKO}aT4Yl^uJNLqn?Oq5Icop?e=xs=^hR| z+wP=2c)585JY*UQQ;G~p!Sb_C!7_vj_X-(Nv!qyr6Rfp2auS1od#-F=lUs7KCtf{* zSA;(b8VDi?Lg~(w?lu}yRE`2CQWDfi4X-6}^4AuY=i~#s@!wa%o9#pPgFHrfsU5b%+hH&VAZ%J9n1KeHFV68o?b3H7Lg>U^1fmqDFakz)$|xo354 zc$hVH=iTPi=B4;VG81(|TRymFwOg84N^I(wK^IeiUl;lrp(L|ThK>$+jJH14w{M&* z=O-|Ei|3i?H?Uv=RIo5GQ9e7}3J{g6kOI$=?M@tWw|h$9Yu)Hs?^JTu?B_p$tzzt- zN>9S}cM{HZ-Zg@`t|31PqrpKc1Biivq0IMkjbjuBv?<`*0T@a!N&eFY{BuzC0htD< zTwwcMEcW;rkjcUXjo`;!%ahQ0egdZ`UJ__QLE~sVdC@6=q!9j%0A_T3AY7=BQLG8@ z&5)2-2v(~9chTXPH-jD~h$@kXl2L%7*l;OcM_BiuOTf2SMe5+vyoF-)51*pMQmGM< zB{wJFEns+{8ANvSWyk9*-ZT@Giz?fPD$rGy0MO+21>-ft3_cXpCWv ztsa(Wtfs**12=nzKjVL8MB+fEFo4$xxzv^G1~J-Gv{rN4ckAIrrT3Y0bM{oyjpR8} zh#VK=5MB=o#gLvfkU3FFgqq5lijXqn)0hg*I#I0D6onQh|5{Co+;=hsDn?3Mz%fM@ zSv@9i=o1h_I@?6?oK!rvvA|0O-jc|Ya}T&jTM~ZhUnasq+7c5uL}P;9l%k`sA}%1@ zF5@HdEH{y(q0&mNj3r=-Z%SiIP#Ej@>8JRe+;oAKs(10(6#c%=zSakZEBY&etH`jh zu+*?3nTRkY)2LysL~mN0cf=)L@1|AiR87A|y-zGYkhzN5C%}(16Tc`b96z^| zwV1ADtNmPy`oWo9l06Dv-=<#clUC+}T3zx!mW!-Q(!JOf*_Go};Qsq5Pj+g2b$n~& zihlY?gh+DyhPU_ZdugL-%4tn$;q0!KdQ*opWR{wi(3WR)+g3KyJo(eo1H0SK9GQ~Y zlB*JevL`<&KWJMy)z;LG)ZSP$e7M=O*lXMe?qlu0+a;Xc8}D;2cjj*9ZkBaH-Cdb_ zdi&i1*DA$eF3+&8rvkh~gzGBY%(uVgZ!7EhW!9FoQ_0Nt^9#60^jsBU!5nb;B zBVo*uJw98Z-Vl!x=M7l&iMb<>T z$IE6uX30&zO@otHPtADiK00n#iOU^P7hxumU6N~ATr%YxZuxc3oXx5Rs>Z0M%?$N> z$+p~u^M%^B{I)kXCAKlWnZ}gHQeA-tD+3$DtVULYmz}9iqDft6=}PIM)>xXcu8FR( z_iC`&ra4>Rm*MSZhKVH0beop0`X4BDPBkO7Un^UTq8C~g5sx_M8lnLar-Y>V%Iq6S zfu)cmxt+QF>#_M2#zDqa#%-G;vDoCQ5{xA7K3kkroJE91HVQWxbs0Sw;h6-P`Ym%! zr#iGc714Q-2*m83?|O>gS6o*>%tlr;EbrPHFJU*bH+j3<*e5s6Hs82uxD|N3^H?(a zz?PpHo{GZI(&Vm>f4AuIg z%V4Y9ybZp>&!mF(V3XXeD1w1j55D@4vP?Y<+$%>uPA zS7&pr?5#LnzR$Ei(Vdc&BB%7rRNmqquEC1^!x33>9pBHt>yc_V>fO@Py|q%Ec{nUS zB$|ihmge?!8fq$Ok`&X*vsS(^*&O;5zu0%Zf#EQK^^N9h(WP`Z#o|RwE4(}M-NLD4 zyZ8t4C-M8@&W4=4wVtWo5?DlbMDb4ylZRgz(yq&>8+RM_Zk<;yzu9DB-hxg4lY9C%EOTA66{5z*@*Jq4#S@}?{(1NWQ zh!f~^>|!6 z=_5}`Cpx)Jc^2Y3!z^?M>MZJs6&@`fKlj5y6`=GPrCF(2=dwtR#mZ5m5}%S(kjRE~Qlem;mPEbed^OXG8gRf&{Cl*q+7J?Wb3rZ$h&Q6sF@_D0D ziNA7HIC<^C?}y$|!6;y79tu(jOndTdI-5@xeDO5z&uUsBoLutYcs%^%)U`93VV)`I z-nG+R6|RR9tz)ED+HvU9RMA%sIz6%NAhC6^D{t9wD9rs)Q(oom{1E!p=e?pnV(Z6a zo}GITK`iHj9pzH1x7A~L)Z*jfva929YqPff%|#pc4g8HPi3K0Euf=WUKI)?2^~Xk6 zLSL-gVvxhqre|jBD30QJme*d|OTu;SW9m@NzHRccF@G1o&Da3{i3h3ASyRK7;txSO z{e@P(%j6e*25;ksW<6;=*H^_C7fwnv_a4-6b14EnD!7?=z=!SAvPIQ2h$Ai%&rTY^FT(?{#|`Rj^#y+tb>ea3 zC;O`h&ujTtH4_=&uP)A3{A3#PN&rziM^gX?BQqm2nE(O+0N`^pG2>AYllTYx^^Kp* z!r9rLhl$D6)s@kejnU50oQZ{-o12N5m5G&=;k5^Yle?|6p&Ns(6ZzkR{5y`Asgtpz zrM1JxJA!cdwy40^z6JTZLX65?}&;M%r&rJUYRdX_R6t%N? zg>)A953~LO{}-~I#iXYK!l(D*+g{;d6<5Q>hLuM1)LYia^4|6usD?w|gAOuv}_hcNst zX@AweikbidAJcEK6+mdVrxpSO69SVK6Mp9ge&`KrpgSFVU3{}bKO`OpTJbD0W;Xax zoC~y;@&f|XtvQBl_m9m~UE`nP>C=g&;9|zL0pP0EfqMo7@1@mG+FYMjIp0fqBvEKX z7zXhV_%z>SdJbotWZbNHxo&^ITD0Ha^EMhnV}?Qk7hMC2`w4>kQ0>jx%mw?A8o=Q~ z1b4Pb0#29T*QLEy*t3$3Qz-A!ps>%Bog&(O9RD{$#&+8og zl}XWX(cExD0ZqF{NBnVMk)?HAA_}3X2nZOyai!0)`8FLzIPCWx!U&JB1VjUD^&>bS z(!1PNnX(mA66^3r7Z(?41B0Bm?3`Az={E1O(hBC>jb=0%tlPup#%U>#eTE1oKkF9Cjv2GUn(n^Lqnr5#_~4cx8Yz$ z*tpNvb+5{}y#f5Ax4j3#!jJgp@y*R1YKMtZq=NMH@AC5V?D?el0MiR^gkSc}>j(saI2kebs6#mS|&lXyj2KP69Y9 zu;{X*r{8N>iS&)T$!!Jxd}eZ)VI5#YL_`$FZG(T06!X|k247%WMuK@QT)h)l-&1u; zvViCDqk-i8K-9n7=nReWLodPf4z}?M?(`J%lX2A6Zn2)LREWdCtYfGBIX%r@ZGNJs zW2IWXDFuay#81^_xF3w6h#?GWtFKVl9?K**&-Siwz1vQgq5YzeS!$|L|K*GN_0cL5 zGBUEvLI#AQEJ`TS1AwM!yOd5Cse6m*hCj#VG>z_n4tCM`#66Q$nHWR(XCg7lMrSqs z&h}+W>R^g^k23c0GgS!#^&a;LIfG+^r@y-31j1Y z+MRIhPksj96CRE`^cNNu6C+}*mG*2u*98uHq3LC2X8!cYiLPQ{DP9G#uuS}1Ss{(5 zQcT+2wJ515i3N=$$*9jbTLwUfgz+3O=~N13-5%%{mgNP|OFiULl%WIxfpbpK5q_pr zgzi8+tN?YKYgrNTI7tlQre%))Pzi5JI$n~c?P4^ zY{m3>EwRI6!rjCGaWcZ-6toJsXe5*{b{tS^LS)>GpkJbUVqt-@s*2_3%1=^505iJ_ zGOPKuwX}>(QnJ2}<~iay1tTMN4WumUmbIPZaj-~;RUnQt3E9h+|R5GnZib&nr=ND5o?RIUSyVPGK%24(Wp zRkgnv0}MZW_%P$X!Odc`Lnw+eJmtrZ22D2q^XHf9m_bJcH-2oV5q5)r4zx}*t_gHz zGMBuu%E&P^w=p;XH8_?@b5Lvw-~F7iii#HHd#^24iZ`l7aC`KnFRro*4IQ%1s$SR^D=(Dvk>&v!TpYryXAK8_0%&`5QmtT3O}*4aR| z66hrX`M|V40Ni|X?lgf_3W%2$7wrqzT@ZNFABc4!ox(>z0dpG5mPnQ0|#k7m-TaTV45aEiZx(6AqS$3gGC{ zxB21j4w|;=DBukp;zJFW-EybLMbpmUV9OZHw%2kbTrSQHAdD1^P9-f zta0W2(ZW*{OlQsrnJ#0ILS}$_sVWkQ`z4DFFM)~5gC7#GlGArTUPVU*$+Z*UqKO37 zq>dwkHt79uU6L4c9AS$(-C3{@`nL-As<=Ur8DN)S+N;<|1$RF9gY7|+cZ0S;G@%I5 zaLElT9UqukE0i3Rney2y-p^9n26{s1nrGM52!VAAVGsCNOUI-Hvlc9r$;|&9?fB5(R2yJ z@%XFLymxnK$DpNsa#k&nBb24fJ4<}C-Gx70=>8;PMTEhDfz!Ooa`~WL*-5X*tlxg# zw^&eH@Gkt^1M^x-TCxqM*+8Fh?o%91FJUNy`(P-NJ(K^$-`1b zj0x#g$v-?ImYRC*bZtfaE!F^?AWmH?d&9^uwTHN(uCmdgU8+DM%26RZ34!b|GTmK{ zzDIQwugdboBC#}$C#K({FMRV_@_}CLd|0e{?`J;1q$v9%?Y&%bs(nuxWGLv-eP)39 zXg0Af|M&9dks65Yi=`_QM3XWf!?}^xmN>KV-%YKqm2yaO=-Y!YU1A%5`>w7b%L8Pg zS00tX;yPyE2-K4f+eH}^>*Gi8$K_-7yFQIRwnPH!%0adabx4y`>1^Y7e#w6m1`a?# zKrn!&>C($odzE7bP1srX=C+BSeE|3+^V++|3lpMiB?<~)=~to~nApzV&Z85y>^Ax8 zFN(A1WtyojB)bmqiURuLza6V&d#Kyqa~_&gM8*Cw6scfqWGmB4BfTstxL4> zw1i`%bQ(W@K;J_4KOhTKis@A&O(PwUX2nC=kRnMU_CE^pQ#|!+P_^>WSNa z?_uS}CnpFUaNf~O6vu34W;R@&{T$KE`x6v(&f_J$wl8E-Q%Zc*I6zCoBF)&c3n48fr_|8je9yB7<6z`1xz!qT=DdzH z(WX&$GcL+B)E{YnknyuF=3%o@F$`)n?mt{0`^AM!O@d@n<|sI~f{@`_5;)Q3<4{9; zxaPr3?|#c_&l}suxnMTo24=R)^ZC*5RYdoRX|#`ic-MRk)vexBV+HA&F93B(5K0LF%=nz7L4^-y!Nal`s-!r@Re&EeJFGR+0kyd ziOObLRzBW*Bp!^u`Z=ndBy8R|PMG=}_OPJCwXVQ0?jgDxW6eK5&yO4#9z4FI40yk6 zD`t9#@$~7m*sQECNhDedjYU+@2>gj2$6smupp7U8p|Z;hCbrRHa-h*~rD&_GS*NcU zNzkvQ)Nu)O@s!UQ+qR7A2%qj7ItBt=$=NH#pq8Me1*F{e3HH%%b98zfRAbGhu2SbG zr>CQY-&`Lre#`yN#>O^v-}j_ySz?KgM%q!SR4)APzNW~Wh9##(lXa`G4oEOl0*#0* zV`vBtCs&f;aK9&`aeP0S{WgMU2=Rc`nd^qbG=TY2cV551b*_FK>adhZ*Y(xB9Gs9a zhUhsP@>mZU+xMOnazh64yMn7SEZYdY#{PN)r6e&*OmtMdTYuE&Lhoo5!UpQWd*)WL2$8VKkNvDj55 zF%%FN?+`Znl|ZC$2+DA+X=z$Y1B;vD2WkRRIDr;&BHF;AB~SI>y8>CWcP3bQOx;GX zv%T{*X1hFV8N}a*CYk`;6nFI|zs&bv_HD)>9BwR6todxe8C(P=#Jk=m^Uq;C+0q5r z@Mr`Wv>31$@Np=|YDQZ;v=_p{{xS+FW9RgeuC*93`HA+iEl}&jc#IFZpGo4S0gx2} z;%2_2l^>ZGvWucsX?7qT(}jz9CTy7#?%;mpBguKStI%DaopLMVd!wpJeVk)V^oS#G zPIJ&#=75{mH2;arB2#V##9cb6vv~-Y!Pblpg^KbFn1eZp1DxJTCq##;1A-N(&J3~% z$#4AFjZfalzA|K{NGHMHGcp10YGlj+)k0~&Q=t2+xtUq%QmxiRVQfW0Zz3zZ;bVgT z#IptWrVeus%#GSvhT~b^8fInr^AWj(bu>tNO^4U3kulp>{5^kE`mCjc zk*3l29Ruz=>1#4l!4U^*9fFk%w>jLB5wYqUjfH*h4oUaJxU9k29C&OM-K6 zicd>^k)cEXa|tOA(zUTH0!bqoq5$joQ(YzQ1S`i|?k0_~gINNmktVB(sDoL<^@o-9 z_5E8tV{Z(60bt4uy+A?oSPv~NqZ+GNYl_A!XH{hGQIn&dX21Xs@pCH&fZ&w8%7~jP z+$tOd12XM+7C1ru=5(8ZWYoh!|JR=U<<+5Lg^(yv*qX1a!h^uX0;fH8p3a{R6$^#} zxlv$1X$?yCrWw@b(xcmV=%!Q}1zL7_JhG1Yk>#4Wm^! zHA0MNjNQol_r&srUoB?~rE1n+herim44*^C8{H9s;w=+{7m@-((E-KtUjBDLkEDN9 zPH<6!jn`xU!^UW&v>9B+IGh~RIV-veu5DbT81m;Z(C?8k5y9v!W!ODrf3aEUqCQItSz5h0sUf&G<|3RA2T2PWL;irp>3xa`}GMfKd zYX39w9L4+l{swWYA82lZ=v*`*???m^b#<3dsG?@-|7$fmTO@K6AD^Bm0Dv{!27@=q z{+p4V@QMWZOD;>wD zN!IPC0kHzEJ5eg1##R>DGXeFr4YRz|6d)}cQy%+e&st<4#Z&N8%;|7d+ykI0qo{?R6W#mcn6R|gZM*RGfLWDyr#+fQVF>dv;A6Rr{+!{=7hpk)Ao=#+y zX#L{s!*S?QaR?SK{3+`ACt=}Kn2Se%*BEZ!vq9K_c}=#MyYcB3plM^fUtkG+k7roS z0&nW%2;_FP=`$ykkYTu9$&b-+oJG9XHS$~c``c9=3l~#V3{fVt5ZgQ*%O0hV@{W7g zp^eqlvH8_oSX?bIl_O;N?xiPornveX!Ln0;X+J=rs%VrD3l}PGnilqg7Km?C2&59u zKAgr01-t6gv$$M2dfepfKRXU>cGMIvbawtYoN)P$A~OFdkSo?2gL7lOa@!N5XnPxb z-eoQ3`^acaQa#7?qG}vwVPmZv#^xqUk5{et{lgvNbN<+~3)e!}6G%e5z@l_(3p1f{ zP9w#@)48hF2rY9k+@EHKA&KSMcPac}r?%rgr*NX0T60sZ8EC6l%(Px~2xDYsdX{bk z1r<@%S5JSvO&Vn!6%R*=;8+IOHz4cmoZutnHL}q3Co_XPdnNpm$T9M$(XzX%9W76G zPs`+|nqe{Ku%9Qp-%>EBZ5SC9e5qGBXd-(nk?>x9sFr50+5?1OY5gHCS>IdNrB?|` z>}x0cet0fv*M|%M>1KE{#Sk{;l@bQ8D|zn6*2K1A>T}MA8AIvWYJv_q>h| zKnk~t90eUcu$P^gSqL&VX5441TGnK2#Tx@LLwZ4P;jCK6xy${0G^b1KSSStui#scB9rQR8@XjlGWJVUi;^IkLW}g@pwQHe>wGkQm*QR!B-B z8U}#+@L znNj&o+FYFidr*!`d0W1!f@XsPrMwLIq)pQ7ikoU$Whb{kR7(8tIWQuf!!}`ILCgKF zZWc&PsmDn?+$&956#<8u{NvRE(>Xs!9n$7?HLlKg>E`p&NJE^}9=Anr;%mRDaB%P- z0S*j%AMncJu$*JeKisZQtripgd7^V!qMu^fDhgid%z%Bm`G|%|TR#q2(oxk~+Vu3k zq$rl~)s2?wxKBe+0BJ%G1iqUo*QWEm-OPU5*k3oRQzB;YAj>pz`5kFm)C$d+oc=A; zy;(;65W(dX`<{T*6Eiq=3gvD&(x`Brq*2$i?mmumsJyl|>_TWPGcE*SS8;Kv8h6Rq zz1EnL+TyJ5PY!qDeU3zx#ZXnP56_=X_X4)Kx6bs3_Q^sa`v3*h4+^mHV9#&ABC&S3+8&9v- zs*&YuNWg>5{Z50v4^05fj!ak#&Y1y^I7C{{kOtQ*1|!a!`_3aG0`C1fcX2=-#t;*u zxUTYP>c9}`9N}q;bVBHrv~56OptB(wA*;>yh0|%(K}F&()3$p!g+o4xL7NuDJfg0y zcF;d?-!G9uO~+h;&~$$uZLy4_@2*yEUkCy=p{7 zq8LoKJDCz!NYYbqqQWM_!4S085ds$0Rep5%{#MP=sHu}{s0~TBY>Nu(#5= zn|rAB>qG!Y2pe7<)4T^~iYC#YpVKHRh$)B0mrwVK{5{z>+OknaNaRN(XNE_UeKKJ0TAeyh<{r`L}<0_v6u z+VSiVUb%g+wpKcLr?mj&Hht1UOA(`?rjOHGqE`HzPAEMTak%$k(9Lt!HkI4B*%~Y~ zXEak^qgWjf2HGNJ(WtTxrPpjl(=#P1RXuMOyqr;;{f3DzJ#1i^-#*l)odQE733I6t z@Bx+honnP5M_J-KTp-FDtWw5>@o~T%H1ryH zrc4fYex@2-lChmC)#U*#h4YkoY6qc*E7RqC24Cahk1Ps;F#%bZLo)+mh`y>ARHc=> zi|wHUX1BxcIf4mE1zX0;2Q&(`2UNnCHdrShk$RUZN2d^kCL%yl2oS;kb|1jPV?sTn zf#B@4zYfe1uKc)n-*elumfFu<@TUBY1h5&OSRuSFwdXk^Aop^w`^!&-{(6Vad9WfS zJu$+IIgV_X!!{#_TWFsBvz#qPThd_`q+tBL&zj;_?Uvy z$ITDK?sr$52{gXPlMRDSJJty=Hy1Bz5RGo@5cCVyx@egIpSDqWO>E#5jI_*-{}1eJBCK(G6(C zwe-Cr%oVtJGMtO93_|4TtO#B(x+DfjbWd%EW|IrE&_0M2*a$t}3;EQHtSE8z_a_kr zUliir>v_w65NfrcycRgz^iQfY{6hopl#c40NKkT(3?p)0=_(OF~F*Q)8b zis_r&_JCUI)3~=3S*mJgYC631liN*4F+s}PE3!lY$5MfQ+lRu}AkpS2 z{UUjT&trUP$ImBQXt>dnliR?0*@ngs8i4{=p#&5Nj;9iW4VUymjF06nw^gx|?D6Y{1c%nXxgNZ&16Fph%rC`&1e5a&BTjaky@e=>zM;f~-aodX?H$g#|ke&NA z^xfqT&Y=Vv)daY*fEROqxkwYoSsm{;2`?I%l^}yvlMa(TL?OUn&dKf5$yRp9C5255 z!4Z>IEu8n|ka|oBlkXYHG~ex7zHj1NW?jbaDqsaD7n|n#M+=Ns=4W0o_s=Lpx;M*A zvRz@)_HzSdgA%Ud)klbsu?4zbx?McN+C9usbf46cwjB^(0kPP2MN`n0XQ2kATK{=CSuMNuxcLA`#;Ou3 zOGP$GK|vvn;JmK?wC(D+DjGesBjC-Z17ii+BzZ{rp21FlU192ohK;9nX`xHjl-II~ zsNFF9?W9pw7jds&9Z`1NJ$*D5mc&QQ$8(3o6l0Lyly?1wthvxvkXq#7^&|{>s=fK;?Iuf5zeiQNOR$_(Vn{w zuy3D+DR$d6P?##Yowf%t0Y)7!M|L11T(&`ik4HX|EDLAnBaV#qYe8KhqOg zd%l?j%+eeJ;^HE&tC8?`RfxNC-zVCoht7FR1n9L1XSc}CS zl&ie%7GD@|mR|hsU0=_)l3P^+tz~Gsm5fxjPqv?#j^{`_?w3zolw(T1exu!*kMilb zYdh)rK3{B|Nn$4Z-_C^raB%Z}+%I%1V9`-g2_!E!zC$mMmpcHC#i_pIh)Km{S*C?D zlaSaf>9K7_owAwahAIjZM~V95MJmnn%Wi?IZI6vANt3&cI{?ypoFBnW$MX-nsi!Dm z!Z-u>^{nU4PC(ndg6z~b;%M^AFUV#7X_VJ-NE~Olq2rnfu5z5Zk2&Sk zaM0NyfI98^c%F`Ee5c{qP0dABtJNZt6n*i$JG|QQ2TvXL5lpPKY~<&gDaJ#a(uA^C zrL|@+DkU*C*R#k>WbEbW#HVP6;kwJ_Z3JB<(Te8RnTlln*Ff#v{82|tQzg$tVN4K% zfJ;~$H-hN0?oH7;ADT&YXa>}aLnN^kpo)2PoGxnfcD=)wD9EM!Qfc*MmAC)g`txx> zZ^#Z(rb2@V#^C6v?e1OHvs{3SRSs3WKiptt(G$C>~#N=SvLD6<6047KzgTp=QEI)q*{#mHe-p9-DTxAH2spXpTMq-fo)xD zaBP=sX-FgBIe-~erwr(%g9Nl-Gu-Qmu%aG3>)fM8Tlk&q@>f)z5i07-yB4cbS6+3{ zUV4Pbk0YFyr%jR(QB*3shDj&e5;?_-afAv$I*5b|ufL85#6_cieF- zS8tJhB)%ftNX8DDn2f1lh@h5=zJ3#oqhF>GpJ7m0_Je^aOP|F~Xl>Qje_S(+2~cG> zU13k#ZRXfLADt-5QU#Px7}eYbAihgpLJ1rMUdpy5c$kn$;h9Q}eNU2VoUPlckAvK+ zvy4}rP7&FJU@$F&Zi;Jk<{lPbH}j1L8d)JM<_Ld^e&g{*cmdbaj%Nv{#5aC?f~?nD z&2{xkvgt95j(v9#)7MkNbSo))p_=SW|1HUVM`^=CRvc-=eooH?$*1lT53dO>;lLl6 zzKw`mwsF0pOiT=PJu^gNgp%n~N0yIgmv+r}pJ*LyJ~I*#i6@IhW3!wa+1c?!SUGt`^}4i=_i6 z&jC_HMGu899v5L#-#ZH2JR%l*2J)Tn8lU&4+tCOwR(hnGbkmAfGvi;R#a~!R3ebg> zC7HcXgE#MaV@F9Z+QQs5J*O{br2$)}y4cw{q|$_HxpVMsFR0gl*B(QWn?4o)8mBFqpe5g*ePl|zt+a`nGMXaG(pOeOhyes(}1XU?? z;O}w)>dphk1wzeOaq!vDHMA@%kx%b!j%*C3`Kv)qJ6nkQ50u@~w#urMI8cH$mz&Nc z+T_ZT>E117e`mDc@K;`4{6G&1#Qm%-UVV1%0)(^=nJu+@3D2$9*OOglq$7V6-89qJMDxXPrqV2#1LVa*y%yJ1fmB{|j$v;}f$$KI z2%R8|2wS#4_NU?8s&FfOiw1{5BOp{%zAw4MmT!#t{J&NqhuC^H zJ*+6}bphVIpl;PZe%h%!BVrfZC^%F<~Ieappob2#?!! zl2YT{#@55?Lq*wYrCt()ZkSTcflD+R)pbcxcB1e7z?d{G#}EG7x%ka`$zgdg;n#SD zKjMatCNjogCvG|^CV2th#bJ$`fmW6+giZv%wt1!tHuef3^PyPXGvD>rstLvSxTTy1 ze3Lk3F%-~_mBl4HK=o^kYk#WwFn$bcT%F|F*bp_9ju?S@c?_sg0jA@rFxg~zpIsy3 zsGAquWDHtdoRyyqGJPL=u>NQ<(PE~@qp!_00;17+0zx-P=W=OxpqR}GS>yO|q)LCRPo!;ihpVej_2{SPyIg1kDhe}rJgrBC0M6>Hq$ zV%P_4A=j$I{t)jK?`%A}l;66D)<_0^7D&pNG6N$-t{)nYd^_RbNQO3&8Pvsqq`w~R z62gW(JwZV6<*P8F61CwLoZ+IQYqpvmgs}4lpH@? zWHhNC-$?22?Sw=>(0;d6lFw?C*#G=HbonYe%vES9#bRO9djf>i8dJz6R*@NxQ#~Qr z$or1%(k=>1hm^uV?mUIV(Rm-V08ypY{No|wy{LW$qA8r*{Sv+1y~J_YTvygAJ{O1m z3hQWSUi54~b)4im6ua&G%HiJYPpG%qDZWwTEYZ?TlJi>!!TRIf#? zP#!LaJ;%~bwOb`b8j-Rr_TVTRpjSS;RZ^NQETb279LnFH8;_R`(h;~=Y7&rr1ET(F z=XjMm=42$koVqg=eWuPJ$EXDLP`K1Xp-1j+QXsB1otCCu=i7Y$NqEc_Ce#S3Czln; z#_F9|Se!#83fydn0^y1qB$dP3mU89|vD*RiTLUorq#Yw8_t!sOLJT-D^5|_so$*jb zcDjL@Dq(P7Rw8I~-NAPGV^a}xF|cVE#TaFhxipuGwm4>4vY%SSSV3W`5K*DqI>WLb zPa?5h>F*D?pJEUfuCnme=BQfoI{DfhUIR%CzfdKe{O85 z=txPb6a3FEHh}ZeK+8dSi_W0Zep`~|iex%6IgLW4+-`HawAZ50;WS(F!2PqIuiYG9 zOM%V%*$vd{hbS-)SwyZ&QsTYM76oRvzy=_AAc;gu$OJ}V(|I^gEPUai9Yna8Kej(Y zNb_YTE8#`bS(_hmXADKk=(kN=QX*dtX(B}Xvkw{MS1l6X@Wjd4$V zUm>jDy*^jcvK`faSHTCnyWq5Q+=lvDISy2}$v`=~-NH|MElTNUB7s8H`kDO%?x$g@ zo2dKeuiO2D=eBZuJHR`mEoe9Imsv?y! zXl9o`pX`P+_!qJd{f}5-uaMt8-b;xDZ`sg#VDg2D3d!n-yoZO|)g|$<(QVh*$^^w} zrZ&DC!~=h4l-@{-M>i&o`B~l&7m+X_pVPWUPSvj*g+HReCrl7ovINVW5({JJ&)0V4 z(VP&Y2<$6I#NV#HLZ_?l`EoWf5hu@{u%cTlJHxkS<<_} z^vL08#Dw`3f^O2x8%i<6{yU%QH##{vD7PMMI}b&~*1hPKmXtSO<^j(eM zKd1H2Bzpk%mnl9y?<(p4j0OC|9EFd+@>>q+I{WSZP`&@A&RF6#k0FJ$2I0S{eSNd9 zeodO#H_->@@cCb3;s{KxS_*V}6F>jg3JB3YM!(5so|>}bc?@%WzU1lpmJAeA`8y5sH%tZ=+^vA|47r*QZi8eo zoF9|>{_CHK4%;9HLlQ*0fB)0vH~Z{WNQE%O^QPwKOHAPHD43byzI7T7#WQh96nITk zI{h!|xMW!5EZ* zDnmok82I=LcCqL2m6i4-Wo6@5wQ`XO673tR`KZqU-fe*bf&Y)Pw~A}4Yqz(H6)6z3 zP$(YU-CcqdclYA1!QI`9Q=n+E;_gn7;;zLlxPN)JzkB~rzQY`3W#;~^Ip>=9xW-sQ z=_j-Q?k4#!+TG#v1=N^xw9P%!9Vb)N zJRL<83JMD-j-8F!`#)v=XIH>9l8!V;$c+2V)F6U@fB=TQf=!BE1-}Qg%Zj~jl@vW@%mj7!;`D`M^ z6n%(K5!MOxJ$}lKZ+!Kp^=NQbB*X7@$FA+Ie7hVTD?K zj#EWXhibf{v@)r8I|=3zadb<^SUFNnLmluTXu0Qc#Wl4ri+R73o?=(K}Y!0r37<{KqqFQBwtS_1{;Y@PT~d zL+AIA8hNgqr)izJ==IuEIh^s{Zqpi?YrkfcZ{XJfn$Y{PZ&J&>2-5ig^d+??_4_Pt z=uzRSAy@?x-7p|5v>TJzT--_CV;~(08_ZRXDSArsD*VcTs z<~=(v|FKFAedZhyca*LvlFtO55)sYJFH^}gGO8eKW0TDoOXux`HnR6zHtEuBYDI_$E4s%;X zCd@MjWJhw7VTpM)j5T&o?-{pSbN+64YI&ZjLbEDico=EvbsIhYnJjPb{js-1LQ(un zvy9`yQVG9B7sC!c;?;jse*ZSBp!E~6dq0hY8l8UDXq`ZN0Bxp-2zv2Tl5!|)+%+z2 zAti;{%&I;;=3j_|h^6aw5+fcErLSjIi;JJpkfX90j3e2L+|N5?Els}a(UB=97k!<~ z6RmG;zg{1v~f0pq%X~yhvI}qP5>F zmkKRBH@Rw1?-CF<_ffrRzVUv(A@o-uD=)#;NT${J-q^4+(!fD~t$Q+Z*+0?1=JfOw zi-hDu6-M!6N5v4=qCFVg0=4oH4YKQfBBmWs#^x`3XROoamAdNq;^w?FWS^kx@R0KS z2TzEZThAjFRzylmU$r8J^&R#is+a`U+|m-IQ>{}HMjk0z>_naQo~(A4{m=P=o05hG zb@W|gl`XDHLu&do=JeLq_Xv??ZWxeQ>C#0# z5?!5o_G7=Nptq>0kigs^IJF#Rml8@cXCX!&9a|^!KG;Cm@1Ud! z&X<#x_1M~{FBy(0IOoPcIpdqA+)UD>ENDps8C5Z{d7P7nDsZDgi9u`B{rgX!`cOZa zUaysDjbEd^HYwsVIZ1RF1X}Z6%TXN=I{G~bDp?LoKTlAfse@uQi$hUA+w4k-!Z-40 zJf?)ysr~)mxs-OKLVtd-nM2W#ImFP})L+LBD(ua0KB9N&uLB{gMoI zOG;zh#~j^LxZN{d#kbH#!WB#z8yi1*SxgesGBT>G)Vog*j7&@+QV1=1>}J?)V|wa?G@mTfJNFEaObGV$tMLP#BfNAy=tLmweT%i7XBE^<|VNtFaaZ z9gZXXDgX48l4WQ|>dY?5F)E#*#^E%myX_whLTJLAX*WC!>;> z(!sV35gQLBdV%~43Zpe^E$gzOxNiqkk{Wx3Lxd4v)E-l(>NNaiU7S>F`m6sM@}&o|M4%ZR5o3mimMdw z>v~)n%Ne?aqKcvhlM+|8E{4EF_M$dQP;U~qwA!GQRkdKiV~ts;gxt$|TTvmCA*`u| z^y;;0@vmYHy7l`Gc*zMBIaO)~$uBSkBQ+6}(iY-zSggoMX+l_x81)(2+(8$vaxOip z)&a;-87S!Rip&@t87Ci#e_F@8&UP}w7)MhIJ-H&IM5hh*#(&>FW4`gC|`ne?}}>w7HHu#o!G9RMYLsyVv2c$ZQyJes0Xuii*<@+ro(X+HnC5MPnN z3_XQdE2uZ0IlOOMRb7KlG@@w=0d}Ch^Z>uCto-xTOT*=>@NWWj9StgT&w;@~+1u;G zr0f(LL>FgDa)EIQZ_7L>ECQbGot+elT|6AR#P8ht)<;0(VD8V61yl+J{O6Ni%;Bb3 zv|I{TIZ2opIWJFu(bKml_6Mkd7T-TG0FC0KkRI`AxlWSXJ%{=s0fLTQKDYB(WS&sU zZ*Rn^B$v*K<(pua9lvyVlSA42`P{3?HJbh0U&@F&y1M)ae-U9N7ux6N-DPr8$hlV? zbCM`C7Z&G(0fG!BpkZh%#=mC)aMqCPJ6}r9#B}Ju7-axeqwyG%@?M|azX{re>467u zp^t!Q#|wOWef8Ff-|F;}dQU1DmVA01tWUqMYy}1*PHRLn=(kGv-BfH06JxL#lvHf3 zKpQU)`*umB8JHv4ynnqCs@l#vK{CK&!f#&8a#KhDb0@wq7Jb*sz{tlqhlNkTSo^Cw z=}6Ndxx4fRi`^~MwQ)WWzsF>jDXIPOPm+K~^0jNPPQxw=t@n;Wnb%%LQvEi6QscJH z#QCB1oO`Pd9nf!m2*~B~Q?0uWH<3e}Q7>7;M`ig41?cowJLzdxv*_mm**Cu>)lG%1 zq^s`t$Z;DXulay*s^NO}H~An6tXY&UL4SOghm_lj*NDIQB2J z-M7gX_g19Rfl2KLbY)9-8DG{ez_gxcT|EUxH@t8D_5S_|^?yfy0ndA4v4r2ezSSH3 zs-H`jmNu)qW_m?3d!*BWq?eWWe%k9x$>3DaW|UO)$DIG^dKh>{LhzC4LmETsOuq#9 zYA+o~P-J;H$~o27R#exHBN36-N`Z|=y85HdqeqjvYpzz^U_= zVrl)i{2gj)$}&xYo0pg})oUFa-3kKB`fw+pvaJKotJNZZNFdF(pzeJ;J`l0*HLLdf zt53+odFQ@LGGWZ$@?*@B%=|f6cuAyl*^|BVHTp*T`21hTd}}QC%2OkrDl)X<7sUF+ ziknNNiM{SI!5GYs5;Ir>B6#rW^FO`MU!ryDEkkAHvViop)KM|lhZc>)ADC^`=?V6f z<#ijMV z6`;WXbvA`U_}4dX74(LRplA-wlhj2kdaL}TVOk3sWh**w#FgA2N*<@5z{w@x+*4j+ z&Ksw0hOpJNdCh8o5o5bV=IGs6Ro+C}V~}R5^v#W{it!*gMNJ4oVMXI5uVVuNo_9{n zBs_UENK5{xP7JuPL{=P58BA*0tO8M}g}BbyrtTK!sd0C|-03s&#`_ucZc_!+az+L469A&eTu&1eP>1Lf{7(~$`P5}PSDS+>rB$7j4tGr<;syF z_PcBb?oM)0s!B~FOH)(4QC_o+ zr)G6|neX8^**}@&xhUKDTXYDiAqt&iLq4$V~R9J=D~@gHNV@_d3t{Hjdd&p1iY@4-#%`m89 z6CJ9h!AT`fwY1gBQ%l^QuG1pkohHUq+9rI*0G}40&II7YqXG-yhp>$oyJolgY6l_` zyV~*#6~75+n-V5`U;zi7!E!iCn3_^F9sy;@8($ph0Xqyz`~F=sG!ZE?`GKOHuh6s& zCHHnnl#X{D8~dD?1X-VPt0_wO203zsj=PAS^gCYVIN&s8H6aj4$0rL#Jjk#1e=EN~ z97ewbrKYLVfcb7_Ra4rhTP5-v7$I!g3B_!e!*{EFv(wXMFBV@Fk{bvWQfV+6&+I2* z_hLipJa4oumZIj^s9mCxDC)F=h>3}-o?2-X-#s%Cm7zo^sPGE%8Ws0u{O{w40CaJ1 z@1bNudWv=LwN@i7ldDc=X&zBO2TNZ?N_gm^+4m#LD7iM)pHC)k9=NX71Nu9Eh;K^2 zMkJ*(5VavZ<^AaC9T5DID;brB*dMVOI7+u1jq{cO)T%Oo?U6|hf#z%Bmi>b8CRBT- zi1ypAE}RGQldU~^5r{BsY77hN3D%I-u8B0mydV0VciT>Yr8+O4D^Yz<@M^AjM>}fIu4RJp5Gp*# zWI$pq8|EBkbL1oPn*aQ`t<~TSpDi^6vD(qVLC5p+TZ~CO4m=_)`N*ct)x-YC;IH3m zca7*LpZMN=kf+QA3=I3aRy_S#^N^F3je2jL9~$|i8AB^w`n#n$xiT&2p}k$a&UQ)V z;dfJ4K}d8DgO4^XqtGgzD3gS;9@4yY)9>lP3Ce&A|AhCvTEm}}-L6ix;@xTrSOb_=S>HZzP#%fS?;WL`bBA5kDXn$9M}p>$AV`u_=<>`+e4dA?`;UK4zY%7 zSs)uJ*RF2tt6476Dm9UeD3iq|{Rx>zbP{ODBsHBxXrO-qJp#xs$UyJLy8+($_)P5$ z&Uy7|uaZF~>9W4Bax?Q6^&v%-b0Ro)d+{=IZaE>;6N#nuk~QRe(wB|rvLb3p9C%$( zO8e}IUGKcpoQ>v2(;Ka&&P{nPNv*0nL1I{%vL4>Q>Qg#kK1|_QEw@UVq~`!2A89Ek z_2J|etP^lk!o3YdEW_$?Ho|$lBeB@cc_S2K)K0vltqotQBqrP8HA>aoZlc9gdj5{W zsXT52C`p+l`HlbuIb~Qe-I(%{{8I|#yF4^#-u^UxTQ0zlJ?xh1^xiK_R==NuLx7~6 zFslGvS;OFLWox%>ztTrR;ndA$^9Yk>9ev`ljb^!J`o;AkKQT_8eU8lgK4%FN&Upz1 zu@5-0+_gv?Ad|VdrBD#r99gReeo(L55k>65l9VZG;obDTXz+)o7Zyk2701Y`K#s)_ zGOA$q%w`(~5CsM=YOrw(@Vu0GSk8sOw4U8W%0)N4Ui{$TLP}Zz*w;ff3)VC4){kR0AAI$`4&VlFbSZ~mf zuNaa4l(B8{orB!azz_5BFMBzLBp?w^Zma@mvuxIosh+Pu*bQ_-kvtGb7eXWa{n-=# zM@_TZHpUWjqwq8DQ+Yo=PxWWGpH`J)EYXsb=I>ZVB1sDacJWfBAB)T=gW|Fe-~H=a z64s3z^A1Nv0(@W(hC1Xw2B?L~Y4&Z1I{w{y!9lZAAmU*Wkjq<|ovR66wD5UM2N<9(I;+okPU-L%z9)e{LVQz!W}6`fu!SIeQ-Fp%=0R}k$kJtsD}_x zz1k0E_w27m^FrG{y_MS7!f|9EbM7mPqvYzw&fMC$*Eu8BIYk%P&0gp`17cmJSlWixefKs+A zZSxy!^V!}kc=DlNs+i*f>k_HkI*~Y5C=Yk;?{nF%9aVFgj2iztW$E8ko=r26fc^m& z!yTa?N~MvWUukbYPU{TxcgO(gS1)E4##~bNg=kva1GQTo&&WE~`|JwTZ-ph&(`bdl z%kTi*#VgB2sdve`2*hI)Qe z`DaoxI>cdwJC@7rg;icx1L5ZTn;Ro${3W~=id^wxw!>gBD|#B)dltU~-pBdNWO>%` zy$x!jZ+vYwS$sl^i9Kk&WB;G-Hne%#K_wuK5XvbRJ3B7AJZ zg91j|OPE!%1fR3T_c=T96JVGzaL+sFqYMA4E_r}?hWMToAIg_mpeH`#YAY?)AYXHT*NcVXW-{qBmXGK>`hD z4jVIl-7+=$ooEg!KTd7z`d1l-nhQ3VkGWx~XV-gOkuOD}HiUxsWe7%WS9d6F!z3xIR8o+vudnL>p=k%;0~1M2wp5YOCUVW$|B`6} z3n?xPBBs2{yj{x_ZYx~<>i@Mp`X6E-%X_G_84JTeZ~ec1586Y02lZN;AOHW=d7w#! z1pi0Xg!-&wklHIseSJtzx14r|7AtVM;i*2u_6gr)ns^=OSe${81)Bb2ge+e0?WM+j zg5gdV{O;F3wMPJ0Fs~nVaGC+m?<>!1$v5X2#Czf!&Zft@=*gigf4CPkVM753f6;@2 zih4mN9~^vqc9>|;em$}gyyX5Lv?PlTe0~R1E{f=SON?XD*Y^~PTPr1COw#xby(8Bt zVP9F`FTE`V{$xbpbR9fjFx#org}{dg%!xkLXJPH|lFZLvN(IQ_lT^n2R)EILrLM29 z!>~#A*(fSjOGt&t`Pf!A&4a1?=ASQfDUbh=_k=2uuKg`;Z*FDm^gHSs8zTMR4dYxr zsd_B~k$Yn+FFRFFmYe=vo1MK`zIahZ(06DiH(7ZE+#Gp9C>iW?+#XsvhEMvDhpl#tYcDMB=6Sf!r0hVJdh7(_Ot^FI-bSan zaEPiMIU(#2CQPQAl)^f9#1nDPJenV3XqdE_S%{k2LX6EY1Uf*SJlK4Ru&^*$ZEz?D z7pENQxM^|>>%3z;9=+npy!Bn#Zjloo8uIb*ZMLae z_aq6w#Mn=-CuTGi?_1-MMSj_ad&pXx%X$mn60^P z$L%`RyqU{)Z}Y~qgJgMddbG_ZcxZh+w@|0%P#N0##1qQi6mGJR*u+I6m*f~?kIj^} zD#7_tt=V-F6dd$;{@caK;Jc$9#kFt|+fj=QnqQG#%iSptO_aMRX}B5M1}BQL(^nOf zQn(&MX1S{e&@bPy$Hw)I>C&gqmTu3A2XLd{3#356`xiEJq6JUs&1f{%gE{7CV;1pO zO#FvYPT06*A%q)7e)OQL+%Vw?oPEQ6zYH$#i>;kZ^q}G>SRTwU+Q&dMUT6;nbJ33! zOS|OX7CcX+izgX>=Qa-qH>Exh`%kI<>S??RX6W2PjS@k_b@fv% zyQ36OaEhzvJAeUU48Ma;h?3Ngf*k@t;95 z{1Ql2sqQh+ZkCugb{5jptQ~tpj;)>U(4{=;f;_axrFKTf3EV%WJw${Y>7a7Eu<+wuDT;?tJ#diUZ~Zd`+5m}vv2 zp0MrYKlDk_^f{Cg$N1109r6$`p-8+kU3ot6=vIS)*>v3!Hd?x6L#a4jKcaPi{D5iwnx z4}Pkx#xM~A2|=Cx5`VHp_-tE0Ah@JPSzHe z@ZQ=u1zanuvX5;T?9jOX0(6Oo*5bM`-8fUbJ*G-~CYANXzQdD|5(fw|^%0!Bbhsme z`T5&8$S0vTb_zU0*MgK3bx$T`C9kp&ClVCg;&%-x39OJa-mV|-0cxc!BGM+MuF3fc z&CQZKxWv!LkTN6xlUAEuUBjnxpO=a^hF=!HQ>fO4++3?lqcQ4AEkl9I7f6B_FqZv91X%NblOCeyULXDL+HGqmzTn90b9cO6=>{-mS; zzvX^oB`vhc!pDr~{q=RU_rV4CY0J@N4_su<_AX=rQ%`VP@M`0{zZT2}s3KV?X&a<4 z1j)7kzEQ*)2k*4V0k8DBC>MWuPXE2ydzRmqHCK@M>P*(Ui9e&A?@+7ilZ&gcgaSVUJ#p&|cxfr6>KL zMJ3}N>GSQ&vOjww_?)vA%`e^6-HUrt;6uP&PXBp2(=PjTdEbUqn%hRp`1#@4%sDVO zKBY9^(!1<~7Zhc1_U;O-2VQUT*H%Px%RC7()T0U-UbM}__`dg54g9J+^W$JNhHaVl*!^Anbgc1x8lnp=|CqZ76CH)mB&&*4 zE_QX_%C?@@4LDZDNtCdP_)079rd?osr^XK4{HuLoo)pt0*TKZwr*`$zz5yLc*Ugyo zDfO%JkFOIcx39IT>-LrONnFM!4TN|1oJm(eUWx#;?7`7uh*WoMs~ z)0e=}Qe5~sc7X4lvgtRN6Xl`U#N+gNoLd1J^ zMIP!EFP2M}wYRU*5oT(?fTff$caR_ZeFg>Ht{Rl=hp1MPL? zIuP`|ooLh#*tj&_Dq^r=y@-83e zM#U*J>yo6EXx(@$vtD8^$71hG$TLI#)QUhDXR2!12qA8I(L4tVhW@|=knIXO6_!C>Cr%mrz6 zw^ZVX+=p|InFlzkl*Aq9$eqJ?VSGcw!t?lA)e?nzTVsU4!pa~ILxJzg>iQKfV53PP z8Mu}_F1vEI+1bPP8gzYqUc7o#=ikt{uDiXx$SNxxEnd8El>GT4mwFln!W8%#{sB z*{`Zj(G?RyXdt*H$@!(EVI&QYo~a&OjQznps?(+p*`cX!N^~B!A4zj#1Oj)as)mjn zy~z8AB@>c=(y)w7$h#-0g*VLi%MUA4o5P_TWjOj8NC*(cJXb@8R zAD8K_=gRlhG#+x{f0WfKEbC-m=Iqio>*t)3BILAk2W^O+DwRtm1wKGim{l}SwMiB= zUDz3(EThW`s$^HT@30wh)MjYpTvP@9_;rCSH27{~h|F-~1z1?5f0%#mlf|U`dC=6+fl$r$vTkQ%my}ul+Q=ewJHo1BY&%u! zy%DRvCwbqAh#Vv|@IuOH3EC_FsaBlH1MzTe?!V-o)uboeb?_rP^6c7ynVFf*-zl%A zO}!5>lgZ1Z)h893#f?BPw?cd`AKO5&JtnjSVqZS8bWk6Y_q7Y%-llyOlqCh9HSCd# ziZ68Qx!A)^0gjV-=#+y{vk@D|oUZn^W*Bd3zMwYl_3Dbhi4oa-=whrVPqeLhox%xt z$iJ~%Dpxhzno&>6^A=g*e>s_`A(s>Z9VuxeQ7=e6rl87vQGh-QmiwT&&GD7Q*09x0 zejR_?#yvLQ+QwB$@W8nGOuKL3IUtgN%Lvl0iCe0EngEhnS626++1n-`nB>Lsxl_t& ztF0ruF zX||N7U$#}-1cj_FZg1O8;^!&pL~Jyxlc1>>nv`*)Z=JqR*Wyd4jm=g&+I7x*C^WYP zDzVJV;lzFtN5_PFVaLLBU~u?uU^q!CRFmEvjMr!4a4A6xfM*ZeW;hKzH6(4iCED@vdlTrV`17^A0cn^%oq6vP_JB}pw9Y^S zDb667TyA+Ysv6H?k9Lg9SVXMlIi3AOiHiB1hHy<`i7AphkK*EQmcs!a-qVG?A_W&; z*oV)C8dRNTyyy5blIE$q!O}BxuFJC{d{S70T+f!!phs1KSy`^;$_DOH4M2Jay|8m6 zG#xqogQ5G`GNr%&U+vU~m_G`MP%30($e~$b1!^LxAc7CEGrUY310o-1`a&Z^D^FHm z;&f1GvnWpVu=^u@vpvqFwCl`jw`bpBK?Ono1#nDsbYi@LTXExD)Fk8znlb>DCCqL{ z&FkDx3G{~|wfbfs&nG&Z{HPUNWOjL+7?+Y5Ml{a# zi)e1wkrPOJP+&=WUb`l>QEQ&>eQ9uj%JVT2Cub+nKYafCEZ1Z5+r!{uyuVl08}S0D z>jIoUm(T1qDR7l7c-Je(rCG<+;@P>wCY`%Q#P6}F@9{S3B--`6Q;4WJrPfk{lso|e zCRbhnM6Hl*CLT_gViBx91$X2=4Uujp!E!K zY5Y1sPS3z}Q~mY|YpVjU614SlmStZH0~a1q%DKFqo?FXExNy&I@w&IL(j57xomtl$ zN%(D6oE+Yb%e|K)OYoenLk?oADBmD>*B@L3dJ`2t6--~VEjIUEt1)Q3m-Ifnd^*{t zcTmeIdnG>NHp;i#sGH99JRlxI-sbQ+0cLUB*eK}bdR_ZfthcNQ;$1O!P$g3CfG>OZ z?Is|S_EUy$x$&bBB!^1pRaMS<=+Um|%p|6G`p&8AU<7iS-s4C_iK!!;?Gzs+`5nMR zK=#Bp|I~Imy!*B^q0am3zSmVEZl~6XwrguZ&UOyC9wA) zfS}vpT7a8+ed}e>SinjS;V~c7%VIqypF1V{?zw2-_bOvtW8f~q?7jaaTxP>Hep<#=@%5X)_!W#jM(z8Qj-8IY`daLgh+VN|iFG$S())yoX%?qxW*lg#H#FLIYwW|?AJ?suZ}th01bOQ2EH<~w zj%&ymL4WaWb<_Hh)UL*-2OG)mN1D|*wk53h1cpOo@9-b+zcFoWAXw-eB~Xr@yrZ~hbwUxVm~^iI*Uc!c3R zAs@=MtA@bXFV{;J-R7&O#9wpXCW0(UcaekOSg)%Uy%Nx8gqZu)ZRUBv1dI?A85njX zTHo&D8qHj+qpJqzE&)JQd<5~M*zfL^@9lcP+N_wE zyfmFb%P3hwt^0VCzUpGbMYq$W_$I9c6XNB0uF$p>cDyQnGy6x8pOt+4W6)TGC|6Oc zzfml;<#2u=c}}`c>fo7RSL@`+etns$?dA9MllP$?&}STm)^3009XHhzxSkuBg4`xP zSzr&w-(7~^fxnA!Z`|d2JcMJOtzDx0Tx+&Z9bQ84%VYMr1SghzEWR^#*=P1IWA3_f z&uJeXa%Jo`%^qb>`^cnYmP{>QH^!vDxy}uMbJro7id=~hk=UG&3fM=w`)YS83dI+f zuRr1Olx1>&qvZ31`l4>)c9!x=_WeM)wF1eW!WS@?N%Yls5rBx9&4RvDlusEIABC*& z!{HDCxGkOvnCOBhGxQEq3vV6v62}$!*GKjPY$s$=sy`;39RQPq2~&s;Q-LpSk{_#T zokQ&zwXfU5-|N#Yfh6rmL5c|$+gP8-GWF7QW-|GHF*w^dfbrX0oFVa zH*kqcZpL!eRy12dXOZD&4T5bR%c}^Q5oXs;L4%xIMB>7T0HUpe&@8UX>zC;rs->j# zo8RAj0#j4cmcrCO*u54nlnWQ4k+*s;pbu3v7mZVRj)G!SD+SUy)eMSwud1hRR*u(u zSg3sH*TfD84XO|SP(R%C`vE!ZPeMKsAiAu!!3XYnU7l61{aPBbmN{pmjzkERB&gul zk;&U?w0a>(tb`iq-oW7#87`56uIF~^3&s>f((z+IomakkzaD~YXEQFjRQDI!Au;Ps zeXA=CU;42o=P5M}sG0v7C>5Q0`o}T;iSz+0 zayhJht!C0SCS0hh)9re(>q7QAl-hizh!><1w`upzXyzq@DKdwA)Cce3a(TSMQX4$s z1l`nky2sO!7%K|?gyD3|_un&7dU~e#ZxsglS-tt#xkiA=`%cJ z1dj21cQPZ8yczqQCtXZ3^VplPwLq&o5G+yv5Ea_HJD{K1Hm^v!sPFV$RRVonuZYcR zYVFZd(>n0H92?aQ)s~S-Oh&NR=v)oNdkY+(!xW_^!ligYG_`u_^~2K={ zaw*3}4^OvlL$w=Dz$ORBT?Dz|xrZRYi^(oh3_YxQ^KD632oMkGz>__&mE9IAGEXp5 zq&R;Y#_XDdNB;1*#i+~T(dfuft6s5i??(P5F_b-NgUP0qN!}L{RVJ+Gfpk(BFSK2I z#BT8$(iwEQJy5;4qbN{?F)TVFpI*MOu3yF5Ut8}1m6|p)!e`vUIb35ELyMMEoI}w= z4L!*_#N58d6ys=sm1VRnK~${o8Q_&Rfss`mebYQT=B8lsO~b?zx0=q`wZ~Zh`mMfj z-U4diZS4?Ali_Mv_;r2R1Yoe!_i;HYmHu?2>&gMbX76SKYc=@iI#SWEH&W2<4D0pn z@`a?e1f6X`QNm|;c!BeR8`tjz zKQDK&{$d=+{2GfS0l&@xpL8d!*wNRx3z?F#pScqT#77Bu z-0;}2Q+l-J{v=0z*(EzFwz&TVxm~^Odz<0TvSqvYN+*@E@#~UKxtO?}r8{Q=7-fITNdGr=qyy!V>+jtF2{=P*H*C(Y|YhK;C zTNgH__1AaMyV}qnJP`s{cbPeRvc9#RYL~UJG&~(S-32NtRa-}`!KVReSNnVuR0yyI#}XLnAm0`TGruasYgaWDRmrvUu#iZyD=!eGQ#c zQ-5^Zg^~};yaj*>h8R}8^-}%0)b*j_Xu}$kw@qOQIw?M{2dvgp8>8v?WQ_*jXy3PuPOH4<@pFC| z`eXh@4QBs28TE(7LoSu|p;qyR+uh(dtMmKrw;j_gUa^kr+^;;vK_A-J`)lsxR*d9 zmfO_JSiFGhL)4n1mj*5yF9-RMu4zP?D>Lbo!)h51{E>76uSoq?qfXQYQMOs!-bTA* z-wVO>dgx2CdIdj)^zxYvCgirFQ?<8tLDK zhtR1|nE>q)Z3h;(6h$kC#*3Hf&OZbQx`^so@cj zZc?*q1s6lvy<&00c6f|}GjeWEg67{oR%*&!RHExfU`|gK5ot1YJ?5i*ImNf$()3RN zt!qi@^Kx|WXrlr=Zl9#Q@$jRcx6=<1uIKB)7ga@qy;m4wZ_ZUxC3VAuY@RiOFW1TT zDT?3N<}@m`svPS|1oF7lceAnVu<4Mh66Ctfjr1vb|GhVVmKFQm#HfMt64fM-zC^|o-{V!NC{^*rTr zSyBDCN;AxRj~4-N$m7u|_HYI{7M6g{K|>*BqE9@U%9DCpQlPh-`NGlS75zKz>~x7Bg4LS{^9S^F;;@6Bp2y>|Ln{j z6Y;<3L3}0}@lJ}jeqD8_{{6}#FAbRFUWd72?$R^7)#R(~K?26NuojL!JzS;i6IGNX zLO49;W5I$Iq;RjfQ|##;x*hALvE-7J#Zf2+-x$Osh&q4A6A4CWISYDR{~^N9D4+b* zC~XisQ}^0<@i6oZ9@>U&4<5a)McMh}tlsfUWwP^(Sd7bOm!={GbG)`|;X3B}j==4a z>Rfs~cEkin<`e2>pm{sYb)KtWbM=hSbtE&d@gkFVgaxD5)}dD8J-7pKg@ff@;C+bH z@N}c!NEc0)M6SmG=Bk1LL=jd(0sO3Va8mevgUK;yCIBG>(Hi*d@mhwQc3>!uSeZ?&>>? z#^`FFk}f!2Vum`I6~XH&qGCs-qo3Lj)xAqRLpROOqx8684*`aF!-SDF?Ovhl zxaqfZh^OR}J6_Y_rl4bv_els@p8H8joi}r7;ER!P-^cETzkCMmn(P@u_U3GKX7Cbw*iQg^U(nXIVZ@J*CFJyX2X_m_O8K&ITzj_GYE zJFZys*xBO}K;K_X0l6t#z%=fLgrk2*|CH&3LI@{&#tomGO{m{p!~QBBPJCA#Hzv2+ zej|6y{ch54x3zAD)Yr4(#}E4l`~X+>&QFDU{L&7AgW4|p8vzQrBVP=}gHbh*u%ZzY zR`1I;EJ9*#u(exGEj$*}-yqDZj@!p5?>W|^i1^(tUuQpayK7eI?YSXeiqG#0-Dy9q z1Kw7c1;Qu{uR>MnD7&2s*%QEy$zEd=-F;a;sEaHaVv?>qpAE3uH91iVAQ;~r-7=Rq z1Bb-1m&gh~u|(~ipv<-3<1 z;k{iG-)B~Tgol+8;~tc$7Ld37IKr$xOa~tlDi(F=P781V!iTfyU=gpFWnb~IhGrjp zn2$1dpP18JYRc=si!Ul=p62?lnxni8dOrQQ6OfBkkJ?j+2* zq5w9Z#P@EwNhAU3@t<2S(v~yqih{)t%p^^K4XB{+)@S1X_d3}tLPg`Th!lDQAg zAKd5}amk@Jf~@a;3dQtQn{2s6L1=pwB>58#3^hB{hb8MiFkzYtlW$vHft4v!ZQf@6 zS*cM3NcL3GGC%!ecwhIos$JJbt+Cl)>|>XiKO|JK#UpH=_52N!?VyQX?b!cidcQ%b zhtL1^Dwq7S9UidmE5P>D42^oz?}-$6IWAI~?<>+8>kST5`Aijp07m$~ktb*yR}wdcQSbs?HPR-+OiNEUmqtToBxlkuMCT;N!kv<-GT)eG)Qnr za0U+$9D-YLcOBd%5L|-=hu}5@*T5jbebC_UHb}n7v-|G1yU+eSbDil^U0q#umvr^b zo&Y=C8k=f0g7Q8C_!c@YcZ`yr5FfoKMzfIT=7~BlfF~DRjgqY+$hr@c_1DdD9Rr_z zd*-h-Ua^$tZRGmxOu<7%^;){RFpv{ygWoG%fZ|;AzJ<>X|6Ld6v z_3_Tj>DBZp$I~efM)QODdF(VP1`@

NeC?49Jg_GPm20>#Ig@mcC*-D1j5I5 zCYm~QKS;qtOtUV$LYF#Yp+q{*Wr#~Tf7Bx3kP~nWYQ5a|TrBy$77+FQupyf0+g%54 zz`K*TeqV`=H@0FFyG&kvgf&Ru|K5-0vmO1A5|AzedQM5N6rd#bi$0fYh@#}n{*hrU zNMnRtzuMLS-r)2m^X%&T_VVKqRlxha&l-sV^X}&;uW*;iqV)?_Khf*UuJyVjDfg=t z9OK6(?F8ODVedJVcoj-SIBgt{FVUk(%?bM{a_m*dgW14Z7lFW=1!btu>Z8(+030yo z`Iof?>dNs4oNYP##RfS{$%>kc`W$6U`r5#^zFUaGG1|(QLz1C_Q?%wzb zlo$YBL(9Lp@ytLQ%93UbJw>|C8<1ch47%vR7DwKlPUaZR_u1v_t{bU$tn`~_jjx&u zqZm=DiCG4@j5fawvpeSps*e2F+5UBaX%zmY|I_dq#R?8_ANBQp4GJ~@SvC^D17Of( zEO-I0KXhq>pxx6g;fm0NE_Kc5CIZy@mRMoGqi{Q*P$}ibXB&PP_aV#5!XM{VVr|;K&pF%fz~d&zc)9Y51J5k&PO%qh#kHJ@%hhV6~ z)Td-WHv{nP)wbtki#rP*7mIU67s}*&Qafo-^W$CaL#^L+s@{hZ1wa`|rM#+WZ^)I< zm#IkBZR+E)vf6A6vX89Jt6|QPqM(fRBX13}&l+!J;sExzz=y~PXM)t`U7|s?C;O(= zV@#uI*;zm(Qe<9;Fm_+DZR$tj)5yPqe-F}up7dj>R|j+V5&SE2l-E*aJ}r-r8#IQm zG{d2F&e_|aN19F^$Ci%~bnAAd+gV;5SK^fjTyWH%{JsS6S)hL|U+GxFHq4PBg||Ma zPs1&XN0M0RXQ{CHl?=ZV`eVJlcI#3*1IruF&^uOfVnA%KE*EY_^5dF}XxjWU?<)9V z9|}ik-C$M6di$4q*COT30BQOlf%g^X@C?g~`_X{qb)*PDGtfJ&AcS#6ZgB@RD8nZx zqIMDV^w@bkjo}p1K-yJ5bh3Tn)K7B&Zx0Hf+dRb$Nq1`Lghk{=s#qawlYPEKJ%OP7 zbo~-AI0HnD1_@k#KA$J>lM>#sQ5v-Y`K1n*|1d$e)nd~q>_YBHe?I0gRF%pYVabJQ zGV-Oh%i*#6=;^FhGy2UX75Abhn_h*0^&C{IaXXZKw{GFiHSzp(5yzZK1mJOfN!46KG z_5$ADFTXYTI5%1eoU)1Ibhx!hF=Niz$z=u(ED~Q$5 z`{P6-i-&k!@G4rEzhT4Ldn2+qWLDTU)xNiZYw!JG`D%xjH_@$db2~;ss8@Y_---YE z4N-^IhAE{8InL3GAcOUbGtzQa-a6+;8>6jwF5#2O@SShea z6N(DF7Vx9LP3Jr~y2yNqdjVAFN4+76s-^uw1lBSmpB|M0QIx-KvI!__ODvH3qV+xE zyf+t?-+oh^nf=k}ZWdoskmn+5F}?^Ihu)N6nl(07tmt)XMkCH5Z?2(axyzN{F{O_y^H@Go-Qj z?qXyTv%|Az3;9I*yTTzB=Y>%~Ml@Wa{(JG1kiHKelJ(5ZQ<9#^K4%#={+=%*n1`AK zJ2O-u6&=NvjNv6fLhlu!Un~R~W;EAZHY>5rRa|)t?ofrZ8U0@Vt}eZ4+HfJAtHLxS zM!n0Tm+E36&5Hum6DAxf=396Fzg__1zgCJwL9_;>I=QbTZCB*UqVr%DvLwzaqh&?dXHvY30Ej$qiwG&F?@uVHg^k=QG=u%6c;p#`d_ z?R%J^Xl;Wbp$W<+8Y1U77;}MS{G6gOPX#=4CMA&zn^7CSE5rFe3R(QUZZU_6-?@0WJnxqbU%s(>*fNk=$mN!|2a+_kh} z4(J$pIYj#P>d#t*rVt7s%35U>k8P@9Y#4CPwhU;)3AAbc6V9!a6#0O|9YajEEV2i(>ecc{_3>*cUm zD{lD1v&T+PwDFxjqkq) z2j;UlLB&2X=>wtNkn?`*^K>|Jvh0CE|%LXD*oyI*xdXB4{g`L2I^p^fvMiN#t0bO#dp38pIHK$K8++> zhY4@j1hH|S(gK2NumM$J#0EOR83?_JGn_;AYqs!$b|uzEcPtSuk+X*O%rCUN`fzK8 zq2DDH8!+jFB{nYhVur9DmhOM6e9A-6alAtU?4MBoMBG?G${g7oqq*`)J4S?2o%RAa z5uFzwcCD2FzMgWTp^wmLHXwA1d1|i9lRGvbOrX0uaoTV#as8rEk2Dn( zzU1l=ag*B`&*gYbW}E>2{;s{(hupJPOz9>r>GqWh@Aar;u*9$QXpwLEOCmbaMh5#^ z3D}?z1r1w&f)@4EzSIl6=jRhi;OF63c(ApWy|#mi4`~RJR>tWUA;SAK(JxSFnzpFd zcMTapUBOteaHR@5;xO;21jr zG$nYhUywqIIrA<6U%nsH6qQ!i%kFy)gbrDmM1DiEST=~1dmt`MbJs^#GusR#TjdtV zOpAwqdn6GEZE2VKw4*P31!JR$ln?4TLnGV}7~)}muu$W`kE9Lzns91$YKL*cw|NS{ zNV-c1K`l`+gJ9c;;GQvwb-cXH6b)@PW=fJ+7OeChkQx28f>Gk=9qZZ=yNW@8FU|pZ zZPfbVL+6SQZPX{>q~)Bg_<^Fix^5Zo|29AwB7}VhVBpv76Kp zAHXx`}p>j4QUxKst|;MnAFAk%&W`1WU#GHeduI#&zy-fgm35|8h0wemP8>y^ zoI}qv<6_WHttpIK3$CJH1k&qMS3yD|gUQe*`%VJzZ6Hg6RRO_T?WdEd>U-+Jtb>es ze4QjeOsHezApFeejtZwrH_OWVxS_aptM!Ztx`Xehc3!hUd|4bqj*@@UV=CkI8_Irk zTft-YM`aYDLh6l)APOfbZ$Cq(tVX2lh6bHuF~qO{Q0P%m@939FfYei=Xm3pH*~;4cvj#HjA)G^P$8EQX4-XfH;9ISYm^$0NekDGQt?}4X<8cKP_bX= z;-Z=aNUm-;-1U3J=cBBy4D8m(63uRgGS`SzUUT8Q12Jd93FC+^_`+&XqF4+XJ4c^9 zDshti^{B0P3<}&azpo*tz9^t%T=#@Ig7+C8`vhQpk~wUq;2EDhxo$Udv$t~zSc_}G zw9;+g%RC%hmqmzz_zKw$H2z1Log=-vws7p9+|(r6Y1Y0;+Zlt((*dV2hG*T+>zFb*cpN9{qAf?gvEb%#cKVPHvV- z8*V9!irMkfi~*t}qyCG?My8VC0X=EMWISyo`GD44m}+o=!%N02h%kMYnKigCt?4Tt zAEqDrfZ^9Yh)C5IoIr2o7{if3$H@W}VS5|1rtoSuuh(WHFBAI4+{ zBdA9&IzAqlA06(H&(w(+)om4JI)QXnuA29^Uc=XMcDTtZ(Sf6VRgD&ex`&WCYO~_l zsD#r-am>oAQ7-6N~Ab&o5`LghPKV2@%=8gXb9V zgDB8W0o0*c;|6W8k&yKQUBNSoYmS>C-}3gPsmI8B{GiCLyXa?Qq&Xbf_5qEVvJ;%) zujEDQ2`HkK==*(PL8R02JK3FI-~FV0j({Nh!Zp-)b6@!uZw*u8QgE}AFEx~fh8?-RHabLWWq$CW@2`3N^+FdYQjA`0XI(n# z^*;#tj}dnGo+rp=ug)>rko^C@{9*r}5warC+`MX|%z##PF<*;U%jHM?k{<> zF!jzCGbOS|>|6X{6nElrZW*!1s2R=s&cMQP!xklP(PVi?J)vAstf`t+#G?v>-QaLa z-AsvH)dMvn1ZN8(6A@1}RMeDb$7>+j?Dkkg1#TXS&vg(=#)YbIxMl)#S4t=b3aR-7 zY`O(5e5-)wL)b1ZTM-v_Q|2y5mMqFTYU3i6dM*ri9B?@^_S4yq#BK)8rjw@`c$e5k?!b&gLdcK{30REt1Iz|e0=?750f z@pHPOt|^1b*{KN!`hudBK2;BH8Kd)#m6W4lFJP0pcPe0>Ci_MbH!RqM2e|DkT*KOu z?^_OG4(xd+W8aDU&x1ADvxZs8V;91?jWb7wmv+*nbO;2C4nP)Dxo-Y&mdFjGf$Utj zN`?z>LJn&bb4@!NeMfhCQe3kWB<_tf&xbWwy)d@ed_{AOu1NKA4 zx6mr`q3mIBBM?Qz<@3+keT%7A$&O>Guu5ZMGc#MpP1W#ZMWxc*W<{GU0f2RoK$L#G@NzcZC-;M*Fj}=vs7=k0ZIlnc+Kw{e_@XQliB|es8dbxz*Qt{(XDAR zLC;9pvtCe9Q9E0;3~pmKgL;rON(wjGM+olL`4H|vo^L$ad9O2nZYB0m=z|I{ zo5YNUu06LB3$zF3j_Yk!Nwb&D*x`m^+DDwj7jW`!UEjixfAr8Xo^lh}K%&`hN7B%W zMQdT)B0ig)YP9G=qPYQ1F%#9|FaE;_dnUrsunmP~8c~C|$N0&usk?RG`EVDC2tP25 zc~UM=xw+xNiG5l=x4@2{YTub+DD=^dY$z-_m2Aj`ea@dht%~Zpm$$xhgjb?jfl|`n zN~j%sXR;0=-!!w>JOYDDN@?0C=DjuL6ey!5%q4kHSjLb)cQ(yejX#Z|@!=<{JRDi@ z<3LltSRNOR{rM_#1`P*=D{!-Rb;Wnc9Gs-X2X$_#95>WeiYFhfeN{B3;6`>W zSD7vUzTr|NvrU04V*sLDqOdG^zvq%ixgU06EDCv_h7Ko05`Iy94!fHbWmSiOCAbm0XhVzgU!K>WC(z*Dcl;Daneb!Pzjt zd*REklr}g`<3|DK>Yo)^T}*Xpkk8O(R<9nZTlWYNch% z{Q#?b)|l2^jo#+u3DM(l-nqH|h3!1EjTFuQP4 zbC1D^gPW%I7V|0ifJKf~yt?JNI*b?mB(#+&m^(eZX))K^O)^S8L!sA#YP{2Oj?izb zv+%FOb+ka*!O9kext#G*c+2M2qH?Z$8qIHg^wKnR>}(TDyK(bi}f`q0IY|Ewf+oQiDZiHiBwQ z)p&xU6OBo7FC@l-WEOe=Ln1RIjOE3(CT$*BslBQVutH<15WsV--mt3NuRw zJr-Own`aak2L6+b7Tu;IECJl!j9HDLxIUe}x zT}eFh*+Lg!MO#HTLW`YnYsc>zy)=6+(VC~gKMJ;2&ak`E9meGDO|9E|oZSbzSG^Ue z%(j(xu-)O=vH+;_=^5(687H{_9)jy3_-wF28BmpQ-@|Em`||BLG5Gojf(ShMUB0ig zg7OsD?--SHoDzxAG@2aHf1G)!`_e=9Dvkv3%j`azcJc87u1KXuZ+2IfTMmrAD43U% zTnOR@5i%;>zNa_FbI;ji`s3^WhkbE!KYCDRhfEyf3LNY>2yp99WmUL5>3N+u&M7->|&J9@NqBK;QWSWtPGSyEs3$8xKUF zRmQlVaz4tF+)ipn(hZi3PGVQAntb;={Q;-Sr&mgZy>fxu+&{okA#r_LwdIGV3)NV+fi1TqkPo& zzkwt#O8hR)+0J*BSz10$o$g1I+(zh=>Wd}0NfjR%o3se~a)h0iywk240V3ySsJS(; z7F{9tUmxz9wSCBknQp#By?6(621yKIVAw&{#63!@M%!r-?{AupQZzK{J*%k*3DG%8 zpS;S0odfhocplg_1ambIl)ou44YaXdD>IF4uwZW*3V)K-;qJXkQ(^866j zvZy=E!Bppvi0>sKW1b~{H>Xf&^@5-q!ylelYrM+jU0Cmxu{#e%=lvQWT3%IF8-xfG zqG5Wm5KN(QvUn64?EMJ4ZsdOCovAe)u=D&0U^L&nIX1{In!ShVo>0uq@y+MKyp8h1F;m= zmxFevRd#z=I<2;ckr0f|+^~+qU-=Sq*x59)c**OM5k1kuefZDS>n*FVPUhy$oL!Pu zhhfRsrw@t0Ojr`C=p$8BjT1B_mv#(=Arq0ty{txCxYkkqBD4iq7*0JG19yfZJ}nfH z2Va8_+gEq_wC^9McVOWt3G;rM@wps2kCi3+loKs?yO_4nD8Jx6_O2EtT97 zsC<-!ZKvU4YNsuiCv#)rk(+BF2|$2MMk(l3lLTIQ^(q#5r8cLh zNWu@vvdzd&E-S=pqMteJcD*QnEL8UzpLjPDkC_ATSApDrK!g=Bq!-2vJ~#8V?}Yph zODPE%yZn!x2xNLD#^F&Q3ex+Vp|c4UM<3g7NdlVHJW&#>N@udh5FQTHILIxRso98U ztuPr3%EFR4I0UZ{K1wlXv4d?!DOKD`udCt534$_GE3}P6-gesjx^-5<<9tAO*K2oZ zpBpHU`UuEjp4X{d@*LKzbRj_A3pAtT*FNa957SckzW8i#IBo7z`mi5{J6$PWU62-r z3thL;MGHx)gN>uJ=x> z)a~rv)*tV^4;lr0Vr<4EcSB2Bv->z2R>&f+itjt%yC|k{=>9M>MAQHx!y3#chydkz(0u_FuH~qt%}KFKCg` z&@%*?3Dgb>zbWOJhbH2mqlkC&{J=F92T&^qilLr$iDk_*9^%3;y|evgNL-yclHwv9 zie89~OPJ@A3AS@5j^?vE6%!VeOV6#oUQZs<$I0`9NEZ{Fzo%jd9d-SK5j4zYkXf&BCrf_TJG-)a)CY4D245y?58YVK^sXXCbA3Br34)i4wjn z5S5+`G3~kGk9)%~E)IsDRgxYWK3E4G@@QcYNP~m}Gn?$|*CoT}=@Z!9Di)ikVhux0;I#&e+s&N?5qw5 zd>rXyuS-~EqHA)EOnw$*Cg$()=4XHjJ1%|qXFn{DT2x+ET5%@8#iw06!y5pGo^8Vl z?K`hBYBm|^a5Mv!gpzQnd1#p*y0r>X8nRfjIXhk-3V{ZP=Sq)O+9Fd}J^rE$##ir+ zob`o;-*5IpElUl}PPVkt=64h5h+MRfw-UC>qR_>kPXy^d;d^VT)U3(@1jAh6_blkX z1aKpBVIW1m$6`>MS3_m>w03|42TBY&Rzfl&Ra*z4vk_Iyw-6jZEVEQFQU>m%gUQYT zpS{QVrvaPD9;7o5WakFTzl+P&yJz=F)?yEu$q^R`G_54nYEj#Yjj8F%8fvox-q!>i z@!8m8A15#(wd=EE0{O-vM{ubj=?dFER0H6n;r~uP%FZhYe8)(GP9aH(!?ffWDsf?< zka4&UV%-(nrOZiQT$EJ$h%;QiRSu<*pQh27sY2%*#K_6$mBFz+3nv)-Vm8twL9%!O zo{+xZ4b0hmdq$GB_#ksG8w#a42xeBNFNc_es50Gfjl=UeM#^cI97%Ji!Wg-`kERDV zR&W5{@q5D;-S(s*fQz!1^i;}}q+ba)F?4o)KRiZINO+z5Y}qR5WJGM@Z_BJwYB?q6 zT!wg0qw-unQ{MKXeBu9A_k(HAeC^7P-~G+-g;pt;ye;lb`Ja)Oe+75eK1)BFLy-T< z=e4P263>XQ#q9`H@T!3$iu!`j5sx>;rCp0ymUv~nMQ(;c4s^pT9h3OZMHtpXw0yd! zSzZQY7Z0BLw%<)lOM<$_cc?DhIIp>2+0#}~>O?n%XwDZ*4JVQM-YFVFOX^{kVIbf= zE`^gs7cKs@D7_W{3CRU{{VPSx^ShDXdo*LXIh+v8@(0yeFtyc)XFkpfXU}iQ1&l0P zhBz-moY*y*nisFJ&`4KYWgwJ9EfuLbZ`3e>NOM1=2G69)@i&4|!pV1-wzl>fQl-lP zPBkJ<#q2;FX$YoXx&UZoBKXbS633N!i3FN;NV6sDWY9nqERd=O3z@XvfdnuV9k8 z-M9s>iM`ZNpd9tMh$68QUB!iQbNLSK=`c!F)XFDa2B?fodqC?XOVwzv6^{PcuTCdL}2m-fHM%?NVJW1PNO?FVrJHw%6{4#y~#|)_c}Q3~yrP zYn#s~dA8v#gdr9vQ&r|0d^q&*Dvhd)QEZnVjWKk2r;>Zkqwc_)pu_E+rKo}B_hM)k zP3qa4LF#q3b|iCQAz^UIYN5?JVBK86>GZ2l;_0OJoYdBchZWUiic+#1n)>edH(G}5 zB9hDcb^LJyEcNzXC?DBT?!Ah%9VoaZbs*72E%W(lNY6j*$CF&XZ^E6m%B|`J9@RjQ z?rBs-6*)r`&gHCi@|2eT5wZTaSf`;fQjMN-*5Qgy#+xaV4)7bOS!s)X`FRF9*$DiA zUGpA&;n0Gr#rm{!ec^WY^x)igA0?^R4GYt&PF~@vt662)4Yj$WmZDGrtbjfAhNV@MZBJKdqk^ zjDMT6=NA_I^vYh)apHXdtKeKlnWWSwc5YOwK+Q0^WNc-HRstfZPJ@nN7mrzk#&3Oi z>~zdLv$DNm)P4`DnYpm3&Zya=Ll$+&+$~`Sdglwd}h5wOJ_>avZg*1wYq#e>-vg(#NR}O#wslL#o^%wpaVog5A z-)ucx*q$M2k5?|^yB`Hs^h#kMn|%8BF#Z2I!H$+lv@G62MvwD1|0bXR{P0>9UZm81 z*2n0S|Dxp2V-pWNTHl^s?SJU=-!cQQ#!*6(qIFlQ`SJemlyyPLf^hJeHJf)wQrKM( zavP>xQgh3PX4vl@z#1}y+>-73bqN`}F5m|{_r=UPRRhzz|3TV=#Mfo1{pm7w4+Yz5 z_>q6R!aSaq{>l9Y=Lf0tkWLCJQGW^0K>M;A`L5H$t+TBafBxzdM(1t{dxBrCHzR$n z_vNd(#N&9j&0K!x{H=%{dV*~22w?Ut(e{Y&5T(FmsgV^*wcB3m;~eHdD)BGFzfW>} z0j(igRvgXyYTx|v{;JX6?_M_!AX0I*tIDX(8nIHT*TgXP**bTlszXU=Na6((;Zp+1 zHFFvRK|{!4hu^)nvvD3JVbuEfhK6a_HM1)vlcWo`+j-&n{d{56#?^-IvIwq0Jh|F9 zV?*zC;eN^s5@;X#Mn!4qcz-Ow90U@{qE;2XM6Z($@VPmz6fF4yoB1(}Ln%Dp@}2Rp zA#&1s_+(~ESou6be1E0A72ZMM{m!}cypoUqt`D_leE%95<_>|r1nw{1|F&Fs`24GQ zR>l+pZ+gEw^85BN#8O$hZtGhSN&(?LJaRr$3H9T!{N9}~^;laI7xLYVq<5tR*s3w zJ@xDwaVHcUinY+6!*esxmB;&b;gv$wg4cF^r+FR?XQfRB?LKi6$H{|)@JvAtAw6uo zq(ULoA^7Oq0v(pka?*^Xl#ST$C)z-f%8op7Kn)7C>35)$&>^5HN92OuGSN-=aCd`D z@^?t1J3HUNtEh$lA;y_8Vws|lXjHV|gtRovw{PEGZ}N;m56Pw~d^cswR~n%r(q4A4 zRH7Cvnw7OWJjHsd&I$GlEkD1Nd+He*2mQ+o>IFC~GUrz{d; zde52MU6sci8daN3BQ%oJpy{dwhi8nrbz#xOLR`VD;=pnAV3h8jf(FwaB}Vp*{unR< zt;t)T>qQtgxwHvPM>;p*wYu5;^-Be!n4lmZUV>i_}V`^znLCDfF?)57`Bx zDJdFYm!G<-lCK21sRii5aqAZ*$<;z5UVEib2krKq1xIgd`-=xv8Z@o6dt8=?n>oaw z>G=D1q@<*@u-m#lJx}OL*rKCcO(~z7ho_}k0eE{$4Ip{$H!5ErRiovyZ~6W%SzZSC zR#S9Dx|TST&Y+lChq4C5^Y8I3H@gl=42o_hp>ya28pU{R-XQm4#f-z1Lb9onLDLVL zE9D8aMxm`ga|EVDWW7?U6(Fygo8DjKDGkdvyzl#U8@0jv&e*fnW3FLD<2=!X*n|y_ z1%eAjKTrRRpCCv(HqQ?lK26_Zp}>#BT@M#Q!tf2`>s-sPmM8V~TNf=1rUR?Odl@KbOs-T0yFk}iV;U51A{H$Xf`3}9? z2k9`#4jQAwuAm(RFQK09+#%Zuccum&oA~*7u6?D!zy4JmF+0*dEc?gbVkhv#rWRU* zTw%=y4L@EO4J~ni(}j%vHZsrV=fzh&swc({(S|yQE7{^9D6dv%@#7vIo`+CS2)N8+ zY~=+VR_Jx+W!!SgppRh!nA;TMUpmu<@UK1Y&o*WrCEwVTF0v+SMT2DGYs^&9nTvY7 z@^OYU8@={i$Sc6*o6fMO-w%^bp$G0KE4?_cLr=>rb6h!yuH?Lm(W;>hpz;X%9JN)e zBu?Q=er8%3et`Xyqp(Zw7PF02X_L@N^QvaG;X&xP^06NhCuUECd4X}Fj}?{rjf*?3 zG#8t4Y#;L6)K>TwACCi8)@dTmi`wmdUU^g0!W(=7V(&nihaj|HhjKCS9cND>ZY7T{;n7t7_-ng`IU@pnFLex~92jhXg7Y5PF^?ANl#rt(1x_4MN0!pl}6B-h{jHo$4v|6F45?|ka zTl#{Xxy&-7VgJf($TyY%6IjD3UBB_J4~e_am+hR|3E75F4RM{OHg2?i8pW_SUYYVz zl=iVnOY@~MD3n5?P(qOQB%>*=LQ0>1tFTNAP<6cADu94V6F^{F@&Y50-BbqK8Dt!U zaVbN!o!U;Tk8As?oB4bje-@I(aT)zTP!WG8o>f{3PQq>;ClGCNxSI7WjP^8D%;)rc zlR2xhs>9GQ_vK9=-Y?B>pm(g}UBV1)(pO&?kuCyBe|{0@ZT}Fu?mjI3Lc0_pW2E;> z#c4Ng$33M|sc(u9qdI#0v3Vk`hLFdi(@!o~`&fY5O2PV<+{R(dnwM0=aZT|>Dsd@S ziOq^4QfBdb^_Fp$W1h?2X*FuAatE*dXawQfLbJ44lP}=*5x_pL{-|vB%{k#U8ZLkn zO^xI_j6^C{7>&qVVknlGBEG{}pCt}ZPBV)4t|F3O18F~f`%AV;1`oPnzUu?|e=jn8 z6HuYKk?R%LAN$4~^&9zO8vqwI1Np06|=#+D1DfE?ay>(8s-3 zBv`z%*mz(cN9>2^ccyWy-DNzpKXaV7?#r+c9?bi!PcdMr{hm?XflyGPKgDq3GYnET!}O1;Zg4- z{(bT1o^yw1oIljUHjkED$2@FThyjAK7Stl2WylS16e%=1Gx$;)3N#+30p)ku#AS58 zwVD7X33BCfpp@GI_(-WHJ)p6^Wszh!(9GQA*U$j`{{N=^O3=9OxM(|3L5W&h|U27|Ip59y-NES+lU&T-9M$9c6Y);ngi?0Fzz@1W4f0^)b$%)(e;f5bp*ipmz`mQKw)sD>{%<@Tu8JD-q8j%r4E0`w5QvD z%6BpYu=7#=$GCb}L9&G_CM2xi{>L6}yMtr?t8Jr@Qt|&WF0mvvL0Qv0a`gV+ykR&= z|D5XD!{pcDyCS7&hW`c*r(f{BDD9Sxt98nl)|ya)rb?8dWl1IewlG2=g8$YhXlQOn zi!&{5d&2LCqCSXS+joRX!HwK99H?7o^R=yPf9*e(`Tr$Lf7qMFhejddJrYYPx?*ki znxO1rd$I%cscS?zi{IGqKKZV*zd)w4{9LIM^!i8Y-}6gk2oy*qo1D8DX_oF+*qV(6 zi|6OwZ5GXFky78(Kl+x#bHs({5;IP>HAjo}bKVf4EveTv%6x8T+US62#qH7aNX0y`KiGu2G)Id5kZA2>Pqhj42q5v+VwLG-U8LsGc*0s=8V#<=wL8%ZeU*&;(NvH; zXcHq>K&(5?Nwoi2E?SR!3Ga^Vp77XipJ+_esletNQ>B*p0c}t{!3HWNY438^S8L&* z-e!|-k#UWUl(ReU!p#mL=4|CV_YKlUfnq8ePzMev@>2AkYx3TT!5E?lCS(WZx>O{u zC90IR)hq@?7bsE6o}R_c{O#qp4r%xPg|wrG5$fzoTcqdu1LqLQ58A>W8@EV{^tCd% z!nn7G!BJ5Y*v=(BWWH^3dYC6yi~0;-Yx&*$dNN6SsCB)m+JF-PA1mv0g5 zc4ps`^P1Hx5=iE)iRR*1b5ODCnlCs$U-KRZVpqQi(ig(J?J*PZSbJRCd(8s7pRxof}s&>QFCoPK~vzqW{56=4~~WX)eoZxfaKdmF=h(r{Z5l58Xu^2|m4i_!C7#&OVFtbV(-g zWtK)N?DgJ4kW$-95)cU?Sw(7DvFm~NF{Z@+m8pWMaLudGh*iF%dV z#RD8dEM`lvPImT(>=zl7lD3CwATt><3;(B z#f5P*aU$)R{SvoI2uO*Y)Fv{VjTGBA#K93C}YqW_{Gq5IX!)UR6^!sa8JiFD}k8D5CCvjx6Gg zz&mVf69*ng#3d9U8;Z4)kXa?!aq#&$ zY~apqC>b%21urU{*Sv0KZtmWn>c>j3ads>XN_>hG)+YQ;3c6hcL%eWtL{!1i=^cIk zG5&rlPg-ap>CO^is@n9yF8?&9wt6To;Vx93O6Z%%;pM?w=OzAfcymYh?F|&ci)Sx| zt>TFRdsDt`mM1{Qnzb&kbZ$`M>WPr&ab53Aj*WrXMy{53z73^SWoL0jSO=R0(q%>* zZl&-9;{a)rZ~`*T0fXag!(b9NLAW8k>{tTbXa9DYxR+aA-T~4KJnFl)*S>8fB{L_? zfI5sb^B7A*WwV-3V=?g14ZI^2<=|MGI0zheAS&%mMLf8(n2^iNr(WVk@h?l>ME=}Y zjO{gZW8#YWEKBD&Mzs8cA4uV3bW;TCFU4e}rdxb-ie6qWvP-2O#|x+}s1tp;b-6ct z8CzX0vUY6|Pj#qaco2az^R}}d-X1k~4D~eU=QZ=9N}Q(%FQAakUb(2zeugfr+k3x{ z*(h)=iRyQ{eRV~{$?rMk3X@xv`@R97&D*y=hpQd;h<6rYT$uDXsUe* zZw3AW)wHVcv%DmPH~Ssu=h4?`Mo)DHoTlM_y5@2} zE75j$ZxlI&5b_$D;Yox#m>=f}ZspV+FSiVrvUnh}HZ%o@8l5L`Z2!rYNfIEQFv7eW3=N{96*Ni|8b_j6H(qzy?! zsQjqKwz?6hJ5!yDrV(I}_WLO}SGTjn&#o11k5Lq+dL#9;3C;xpLU7cO2SeQw;q6BP zD_&UZf6M`>lVv-dAa)x4;O|MVbmXU3@V@IRHZ(A3HxLf!XhTMAdsNKi)vLr>ww@|p zNHibzt4K*njePd8CHI;uoMx0iaN3CusapIrhh|R1juyvDAL2O9PuDHPowe&jlxN zattF);MBh@zJUgVNER-wr|f6GG`0*9j&Zz$^_^|lCr1wT#sRncQfeUs;k~rv$llMy z91;e-Oj&7??n8azJ*+wv01GVaol5|p2lUSiCil~fz+m*ueTeCg`>-Z+u%!aIYL zwuv!kT=6Qf-GWW0|EYKyZbs`v-SA^G4u?-ubA@ZlTWn^?iI&iRa2K z4x`y+kSpre8R#@Y=0W#-!n}XF%wr}+>sF-aszt(dbLIZz#@1g97jn~f@Gh<0>CA*h z)rb^RWZKgWjnw70%PJ9CNTB3*M;fV^Fl3UkrWxDW`S*|HK(okxlk9$jU|iFz`y^+% z&K&Q3Ymor(UAlP;rw0?^VnMFn>+-`%kn0sit5luMW3#IM5*^t_Po^M!zvTa~81QPP z?P`521a1-&6X_+Oz2GG}*?kC>uoZThj|*7_a1IIM+}VMuU7phIB1Kv+oi$G^?L_^& zQ`tO6P;fb#f_w7$Mr0;lCrDsof!#B~EiPLymv*0kO15;LJZwtA>CdV8`Oo=sQ1>uB zwpgm?!HNfm`IwfU@D%IW^qLk&_YEA-uymaDWm zmA9G}+HaOi1ZQ|#^7WBQRE>6I4r~MRDzzJj+5t+kW}NU_+JzZiNT)+F@e@N3(7%%k zR{;6^7C(@4*`-TFZz{bwCB5$SUqe6*I1lBj3Xqel^+C|A(ml$Fyd(MPF6nzWl;U7F z2?&S&;Qn&jU!mRn-rlXT3`U@6G&}flrIO<`zMQK-rg_e6YMbpm`B0O|I3og&A^emd zv}n`5`v2&93$G}{e_vF(OOTKfkdg)oNofQmMVbMT66ua%Xhcf7TSU4W1{jp?aOh#^ z9y(@-JNusVTYH~%?)?LvXU%%wdB6Dy-})YEpeix5Ey}A?QNoIgns7ppEDqz`mZR45 z(~EUM`?V)a&0W*c7`UAMFC`c#g!rjgA{ah(Qih6qU)Zm_xNTOM&yi*!D?z@aT_C1l zj=j317xRUkzM_Cc)Fs5C$F07Xtr^+q5=Pam z)CQlkq??yeG7aA2U7;?semaI@bn9*sO%SZAsWFJ3{_K6;ApGsR!dcD_pb}4eeAzbC zW_w9;>D=m(tdRAB%@OM4I{p6lZmm_H*U=SGRJiB8iV~Kvj2i^I!+zY1GtS3BT2bkT zocYf}b>L#jsRGjmLFMja^YBr#q4hj76W4F>+vojpjC?gm7ES|_ey~=DrxKL!JfV<@2l9zlT|n`-O{96R%7nNM7buI(TXupn#YHMeE> zP08%bTt~T6!?9xZjxhZGtVsFzZ~Nz7%x(_3r^XMH9nPM@ZSI#}I<}JYPuzm8yRH|V z{qSf;^X<-e#u|WU&N8=$9;_}l%?{_broZj5CW3jCH=U6^sH1;~yC@hv6mTtNk%s({ zfIca6^>E#3$ZQYJUgHQg?s80b`sbA;evX+f7?cAOp!bxyy}@u`rq)l}{*M;G3vG5L z&S4>2d@}5?hjJ+krWU8A7L){*sdP=?Acmpz_qD#u>ba&hq~L0cGi>{hf2?fE8hcnP z)WK{cVj!$v6iXvdt~^UIDH4Af^i#jQ#HsA$n8i3pfLB<*NNuBkN+0 zif7n@KRBy#=U>0}h95JgzDlp`{SlqH^lo@_&SbH)=Gn*yPN`*(-+f`&`@e+-E}CCb z)uxQ8&|vcldPNWZ9Bcjllo5QojXxpdTB$VA0vEJ%6S9v*_j~#GZRY+%n7}sKpWt%@Y|e0hq^hV> z;kZTd7$egI{*}%U=7DwU0J0VUA1lXoLB- zmi{(as_RSnJq&bXhuW9`{rffi#FoKec!_|b<2cJP$Jfm~eR%0YPVTYIoEyPDUqgw} z;d=Fz^YxF-nC;QJC???SD(cv0iBenELGE$x{_3x(Ece9rOVkwTao#y?Aj0+SCV+S2 zAwUkPO-9lA;b*WAW=OB1FKPneDjaMWeHx}(=7Z2f-RAlXAI#V@Kb}Nw#i!Amv}@D|8qt`2EJ7`DSHk_63A zWU?2v$-honnE>}tPy^y&1G-t3ZRK;njlnq?$Ys>*-lVYIzEEx6aU{~RT#0h5|Mo5s z09`t6?c1XG=hO0y(K^%-%W48DG2dwuGiC@n4fP;#U2<9I{2LUAI`WR_pD#Hh%-w9~ zek_R~Ix@U|RLSdV^S(!zacZY;L(354ZpZJX4aH%uVR4L7{azie|GxUABA#VR+xJD0 zwfM)OSLlZM-M6-6c>VfAzJ70^Zt!*Xs|TO%ne8vjr-hNI;DboB&NAl5LDl7hfec8P zf4ZxkrieFN=BEz>|JfR$L%GKJ}86HSLRyv>BBdc(Sg^;1tA$+z@j z8}YDu{4m!g=C-bsErP;?=D$08+~EO;ZeHpe(E~bY_BDW2U@egT%Wncpv|8aRz%?x9 zsr1_wtzXiI>O0+SLeuCKh6i>Di{rvv88ORMD-J0Q!uy;meV_J{!PShNCDsf6jF1wlIjuol3oS4&*^ z?^?3X4M|Q1nJx3cn?%*a{O)_kY|xalme%9$2*?Jl$W%K>S25^SS7;K59p*VyF3L(W z^d-SJPq6;>Y@#(w(7E%dV)=`8?+5GPGhgM0o#v$@yvJL>m+qReOwkd=J^TsNN}-!; zz$31?ZimOwZ91Sif7v_xex~I?`Ha+&)VxH73j*r%g0{CD=7Ue)qey5bWNy04lVRZ)Q402q14LYD!z(=*pwR9NVT=jWXP04*CALtt@TlOF?H$cqM%AIp#a+_!4&IX4S&>zx#$6 z^x-u7s&dGXdpJ$^E>rrBMc~vgErx^3>%)bORT=z?`2AeMYsO=z1V+jEfyNPDEwh8= zk-w1dZAPYBbAKH1dbfi54*D;7QhnTOJ*~z!@6VoZu=7G?CM4Jb&t(Wj|FGRWTgm^? z4?Yu%``5a;fnx-MqcvOyRyx}}y2vce3;}k4+ohU?`e!3vRa6QENTQ3c8C;)G!3t!L z@B^4>6RndIr<5lqLQaCF0@&F;0S2LZ1@x0kU?IheetNRb$$7Tu8>h3H@;<}?O zi&4d8YMS744#RY={={15*fJJEKFHbCk4m_?gV7 zSMmI=VM4+fG9!;q4wrts83Q#DB$xUbJ;U|0O!MWoFuzEm1eSB&KY+s#=(yl49*KX@V*Wo|DTn%l}3y|!Xp<-yxr?_`0H-Ee#vzT&TI zwLxcT7hdLaj|--Ot611c0ymoifI5Xp^D25z!0kR%`g7b?ibG9%WNn3z@N0UY*67W< zjZRp&N*tqk;Ts**%_gHG6&?!=NI32Vv!x**AU9w)ZCFFp?@-OJ1iW#H_wD!XD>moI z?x{Zb=+zApq|=YWr!eA}?Yv1t>@kCa)ifJDVQ0ArH{zcYU12L~GE@;P z43!a$6#KgRg}Gx-G06P*=C43izO>~1W?M+k+6qKX)d$jrzwsHKdU+NZ$t=*1>h;^c zaMLfQ50Ckw_idBR`989<;=bvXnLylz9<$*Fl|Nc<`muukK)dTCDPZscbTy9ocH|Z6 zktWfZ)*Uy)ind+x?OXM?*c}0LUgkcAD>rRchwUAY@4ZHvq{x+Z>T2Kq?2wA2_1|@V zgiZLZN}ZpajQ1FsPRzl}vUl=4j>`f7C!O@)+aPe+r=)f^Kk+Qy!m(LwNiJd`q86?d zh82+~PDM-!Z5)J;*3M}$CjDN9EHn2JUd9z~51iv^*zU7g{wEt|@Koq7jA}vdIZowG!~f zD1zs*7*cb81E>TnGfX(=&|Y{6G|}X}bRu@o!S82+$^w3_67% zXOzltm(m{Q%zLvI&Pf^=VK@~uKzg*oCk`B;@*rke@xKx-v7h_Gy-~Dde)c&**Zz~k zn2aw$_wHI9f zm_pc}@c)qCFoy;^bxkg6?3yo}Ju|z^@-lli1eq&`N}txK@EMk=H}x0^(6YF2yx-0u z{EQcSaJRL?SD-ol9lSh}{7MhsfI+uY5tF2JmEyY!@A{$Nxf7OTfbf^u-(|l$**OHs zsUu4KW)=-Nc!jxmcVC9rUR3sbPeccPec)KREM8h!$MJYSn3RL26`GY){1ax0^gj`< zeZ!1MA^qrWF4b6f|33W4fZwoxIQ0WS_lwTw`36_to$a+hf~l%%WaVVolAft(Nd=-a zj*JZ}lo}qkX3APbi({793>yQRgtJky)BV9hR|R8eNP3*?cD1&eUbyXC zS-9%=O&mK?JEH<1JAMI8j@+O$al^JfEMe7SpAnB&r_{lo>W+qEEv_ZxjWyXYwJ~fZ zBv)*J>tD|AW)8b5(>87>RZQ{ItdZS6$qA})?p!Pj3k!r3he!Mj5tg(I+l!#Ag?A@Z_svdX5%`z?%F8=A|kH##=4sXWh=p$ zB-YSi*go}KPJM#&xg4i-Jkm3kzNi+${~8eGua0IcWJl-t;WOI8rE_8^@<%ri1)Qxjf+#mgj*QoLe1BO<6X#nty=@# zyLpYFy8VF9O2Mf@zqW)MHT8jqqTzY-WNmN63*KKb@EO-lk5Q6Y3C#c1`?ipPZLTvB zc>fafp5=qvzZ{%jFE5Y$(oR_x?O;olW%{E%VR{-80k2kf4C#x{^yIj>v$X7^vrs{Fbs9+ql2gQA?fS0yU$M}KakQl z$gNmw{#6@y2t;k=o<%O47-7D#aA_RZp9u1+CEc9`-xHo%cVP9KZyvh58qzfL#e9N< z8Pu38kP_^!c4Z&gEfk+{@~&b{Fj_*cgd^w}Q!QhD_)JC63})*aa@u$`2v(emY|n%7 z!~gL~d1tah9*QFLUZwE?;LaUKAD2ucE^}aotcCW8k3G$Cwsk5$l!E2)yDe5j44Zt= zvZq#>_@`T*?sSxT;!SA;!ADC?E%!Pj0hgZWac_NRY#8ODL@Jp!;WhpSzLCbuKSIqM zgiA#$)O;oxMQ_!faS}hQ9|urz7-+0!`Zwnjy5M0h3xK@3QEI^bm5%fdbxs;F=jB1M zBM%6RRREWVCU^F%JHTDSob z3UkRg`CYHLg5yk^g%70RIdb58)UTdpUKZM#F)0Mt4o`2b&A7?GDdLi%{M%V5WLf9- zWLc>CV>w;|E-b^aQzQFj4}i~wJPo)uZ9*RZJ@}hfpy`Lx@7R%4|X=Xq+v4Vc$Lnb)c4~@r9e&Krm^|PbO~>x z)662paHJ0rJDyd(dmO!}7=nlCX{SR{io9XiRFB8F5PP01bXK06nd`Hg!#y2W>s`W@WNS z@IYRQZ-vT8F*Mid0A;ja|42-PA9vgbaO`dn1RNZ-9L=g!mb#XOF&5_Zg+$f)<{_+og@o zz4xgtv2CFo;L(1D6R%^PB6W4+=Dwa}qhW3rL2ThOmwM*x^?E)8FJCKTcYF_~?TT8*W1j`cq6}ZYuKE{kNJqD4TW9AodC&)Q z-C`G4(fq0EaEyk7eEX@cFcuY(5DH`<{**mm6`Us7!D6)hnZwJN}k6cH*hszSmo z9wXD#EqPO}2^?yBv3Bcs+n(Syq-nip8}>9|{i?VK&eu`9`~@j&G3-$jz3O>bsA{!t zpC;w&1u~J(yMjc|ia=b{;lEBz%_#fh-QG7&aLCB4bd>_1Yf8F0{t6?QPf7R`?Hc2* zJF0WwmCIAW!$Ph?`Zjj^rIRiNc;mJv>lr z^n6Rg6&us$(ZJdHB>O#A_)AFltAkpC?XQrZ-0d~?K+a0fW=Q9}>P(q6-V#rP&$U(` zmTQP`4sBG+ak6ePUtsO8eKYWYc2SYEHK zk}|c?zu*u(iE~EZ0_D#K^)UL7FAC%n`tF8&=%7fx~&B^3Ua#Br%GMLz}S+KlWHmpS4~iwy?gdr zpF{ZQIm2l+o1}2+4|OVwP7k)Nh-0^p;u{29Ccc+z4x=O}c{(E1X2TyWc*}ho{Vepf z8ano6KFG?2Gp_VT|H9(3tQtE1C?BR0#^*8m|H*miDiRPDJd}~-rQt}ToXA=f^P2%Q zNv%$OePEw@(X)Pa12jo0gI@$DggT+Y|?xFj5jP z%*U%nlOQ*D2IR_$>w_Ncj@n6ZiT77{F)|d1YV#`I5Ll*ay0W5|?W_04(=ih*l=BjV z^dmm{5u9Co7*ruG%_$;5*|>#_JgDlGfHA@`aNKvjBz#JuAkW6wE%?vs=TE2cVQkGw zY(Ft@(R-*W&XO3R>M2>C{Xd`LrVY$(@E73MspRf&E{S>W7fHfjIV^Wvw_da_#<0{3 zZI791hI~51zg)-DMU9kqiP=>J+|+TJWnueuzUKM3!}X^s=`vA~8qCd3_|j&TmdsU^ zYxS##{vu!6^|`?_nMUJ;+l>O&79 zq-Pp&y?TO@;UazzPeJP_69tJx13%&ywR#&K3|Ktg9RLW9QJaN+qoU(q$r%`{5%?Wk zfjz1ENPq6wgfPOqz^hy71dD1a=S5ubDCc+bbzXIe`x4UlE?dpJ5c*NOH5u1tPO=H& zQqoZ7ScmD&%wrm#6&8f5>h&R!K*lHA4SnO3mrrpc_lxj?v}1p~Uc{@pCE-RpFk7Th zA){EM6ESgqHd7I9hX8dL(zWDHW(#AdA}KW@xUAD83AJKJ|~n z%j(L#+hyv+N#L~k=RCi;*lfS;oZ{nJ1mTza0mL2zc zubiczESiAA@$Llor4U^%X~4yw31)P1>MFv2PJnlk)w;^x6=`?L6kNp_@gbes)2kd* zR%F`M;l+t$MuHYe%&LOvdcQ7K`|WB8x1-)Xc|%dH`zg#%pSWr*#xO>Bv?aIxmmqKa zD2L~Fa>b{YiGyJ}OEuce`bn}Mf4|IAI3XxvC;o9O%3u&wvzy^w@P^GzyGZMV<+mx; zM@$+uv5EFl7p|GHlFkiw`WhVp=R(KD|V=Ykmq-g19Upc{#ZXu$w7+ z5?k785R>M-U1@9^EMJF>Ybjqqh2yJfv54Qjv*PAkWCylZDg?QGM;A*k{?oxUZhrxO z9&1#tClAJU6ytH|E%VH*Z`Q(_UOTcfHGo&z%Q6YRB68iFkASqQf6hd7aC9$`f&&z z@U|Ku9K*Aw`JGk zbq$EL50ZXrct+*Pe4loh>GRW`u~Xw8KY~adE;&dEWcQnL%w8}mkV4l>v#fbJcTbQ2}?ffB|em=#-dKsz;4^;6e@*O!sq z$|I+mtYky!1bLyL;w`^}^Qdz-W`4x^3BQ@@g{(#xyn4`@u<_)>&O!r4?%E%xJ_|&*9SU3lki?3Ba_W2bhOe2!&yPoE zlslA?m>;25AmaR18qArp1+J_lU8caqB|#!m&Iw%9zjxyIKzGhzoixL8sEQhx9F6DL z2&c^-&UNq8eqV{Lty4PBEpv$o6ypOm5RnheJ89@HUA51C$aFQtfGC)eSVnhp>8=Sf zQ3J#;!{L-c%+4`^52h9}5Q;~BW0{nl0Qt)sL<-4HA@Oe1x+dX9?v}==V(XE;AJ1

H**Iwd)GppU`n6{S<2d<}t7yAq$W< z1jsI=3dhcXcz%;U%I4J62At6}SpfQ5o`EBm*1JIQ7JW95+akn$g8(A`Ui3~j5Yg>% ziRd$2jXLcI5$){A8WZkpNlm;20B!+efGEpEc<9?#2m;}4K|Z_99kqyC)?MF{;}LI z#hUWXnMH5ALNnr(UXUQbCJfNbAHHQxu@<->**$djn0HS$nR;uRpjq`vbd=}hJ%rtv z=RX1VzlN^=)t(M-LyST z3Y!YhDgR!sms_BwOYn?&7Bt9H}0N% zS}N9r6oy4LSCPp~-ZYoPbWjX;rP2%p%24W5v$G;aD)L{PqUf7L+J&)(5`FJ8C z*Chvom+w6U7uAOXa(PYgv~_+~o+&Zy?%U5{3u2q4i85lnR?=kSf$)yQ&wB_lYQ zFuTQ`Q9iop(KN+BaCKL7@ryj{9%cVfB+FS5<&!fTWzpAM^4ahE^f))S9$4Xd%nV2z zJ6>6A0m$gVR3(DEQ)W3lh_=vfk_GT*br)Pu1p+SKn1C*BWBvuyyMq})GIMwEK)}z# zvNVF0bA#2WB{CQ&)rsl@RD3GWr4JCZn4X+U1T{?QguBC00(t{9+?3AAD0@cq<9$G9 zP0+=gtnS*KB6RS*ynZk-qMve!Naa2>ZDe|6;G+y8g`w`u?gP*fiwKke78v%B_oDfr zDgn9y7u-d3J=Sj#RiF-#-sQln+q=tyeX`5EFZ5G^Q#-EAL@4TU8gN3|n+1>7!}K<} zRPQ3cqNCiyfY%`T3FWeZoj8kPFPe*$?u2cDeUHYxPthA=#fTV_nhB8l_gTZa442p7 z2b4lVYfwhZH`U!o6e-8C@&EmI#gKmj=&iF>ezSbHX(`37yuUg3=gmVhOYpkq&7iJf zmAZ|gOi9-927nZ$oj1{n`0t_H|E?o%$tSgs+Gn)oR*sh2Mw}KJ7oPFxvOaF4x5hjx z6Ts+v%qv2s3YP7$hg;&2GI_R{wh$~_99Y%yoORoM-$?vVg#6!A^#?qb;J&qQ47dib z426wpGOsJt>sT)h_PM1aVm>>OGKVdA$*Cu8?qE0sXj#VMx+uM zShaxp9#_-j{!REij;Ejc-HTV_be-D_Z6u=$kOyGBW4#y<96F2=mSqMKb)N%Cy4h+b z(hrAy^5qtMz_-_%m(l35XJO6h4AgfSK7vBI0+&&D5BrXnw?J56+ett^6j1DSb8U7n z4!VRvCg1^p$O=SQlwy7TgLysAC6cPDuY1Qt2ONr2rUL+4zgi(~RbKrzyUDhhl><8u z-=JJ&VRDaEwe7%MaHqgVep@6gcNk@#ncT#Df1F9PVV*BQ$Cy5(e4i& zHGQCs=MVi|6Nnmk=g@Few+G@tVnGH5q{o>HsBbTVl_gVEPwYfn3=K%>Nytsex?JSh zF#bJK+dJ8sN2Va=P3Z@~ zV{QvN*URQ)cfyhP9ZlRONk{DleAzUYU`9ZO z($5Va$K+7;o!4FwHSjxG?TcKHY^viY;=i7=w{_NM@Ad>-%u4khsLpcgS^sWV&djoq z5$&rC@|WxREINb)KNt*f!NS zM+Zsb&|Vor!?zTF!)Z5Bcjwl(eiz5DgvK*@<{Jbw3=HBW@Ti>~}v+OD7IwmD^a)9;-l_@(LOH0YCb& zSz!Ir0%NCv0seN@voW1_$TTQSNw(ug;lcdk4RRHfU=fp*a0iaJH-5}He9lkVj%WLm za<^ya6b!Gtf^S3o2=?gQACugm>#N95s;zj=8%gia5{l0X)u4#n?dd; z5bU3)U~xbxuI_g+81|}w)Si!S;Gxs;FM%y}k z=T??8!p*{>X=sg`$r*5s^Xo($#N0(~Br${sNYhsCO898_*3Z(i(Y)t+qx{s*6NkfU z^_>h8C@BqIA^SY?VHCCY^aJH5_7+~V&dfdm7A<%nn-v^F5(6#7iOqmV?PvKco zGIpnHe~o2XmG?UkfBfCqRIF*$O1peHzW@HB4@7p(J-ibqL_1AjeNmHaVIV7T8vxo; zyjiu=yc$Q>vzqd-J}>3t5F%RajHIhR=-$Y!ETJ6O8m04T84T-LH)1i1`f)H*LAC-q{ zd{7fA-1~U(NIi`guAchJ3(^`gf_9jXQ`1P}^;_22n?bL^%(=V7v9Y7OLw?%wO|{&k z$n0TgPei`0t;9Z6GlKr4U7pd1hbUB#5~`APZ)K`j)!ZR2)6GM>XdI1NUI0cvl^m0s zqOOI3)`FW13j^OcCbV<6Xy3NAoFPj!Glf4Ubg7+9$VYH7eUwXhH_2_@-gr6JqV26f zt8EzY*NKGysnqR_Sg`iBs8jIU3e98^$$5gC!VRd-;VEo2K~5}_llq;ku~bBzsixpb z?EX6{;=gSBbbHqPWI8?Ul}|=}c~@osTl}r5;(>&^SfmC${9$^>z6iKDsy&to&Uq494rX3YdcYYaVpP;j`*+Mc z7b3?f%&B(5VP=aDXVhmu`H<k!u(}rY*ZvjK&7{kkACpqmkL4vzg56k=& z{cGK9Df?E>yimpbUbEqwQznTQVZ-$GV(Bki+Kv4+CdpUv*XeF)@4=fNdxCHqut z61=;$+SKBQBs6cWED33AtaZaFt#b3U(LW3ON%+gfFwQ&hs{WvGOE?TrPAbG~08zVP0Q;shRXe$l)O*nQF=BTND zI*X368%qtXvJ);Uzo3jI-9D9BL%6Xldh77tc3b69jqAFiIIo~PC7s;~S0UGMpSAER zbI7}HmA8saoIi3gg9PqWo5@MfTLKUkBl>j;IDLrZsobitB2IhGP8A(!p&~|5n4Kkt z(+uY>Cc`$JtgqTWL%1mMh6brc581}Mxo>80bg_Z zcoyRA`UBODW~!sug|Y6I`sQTAA6NunzhP_XYks!)x&Z+p>HDN>&nDS6BAJbIUj<|# z`<|;a2h$iX^0I*kZoU%l(>Un}Y|#;Iyigt(l-jqGzQ{*NW7!Qe>zE(%a8BVZvI69- zKk#qe)qDDXVr<;mnUmBXl#{LLj%+{%MID?Wc001(6y_E%ec|>s3smODjZceXJZ7E4s)1RgjXAa>!q5_oe(FHi(38593D2 z&+%IFkXWX{7thpG>f@8Sr~If36`y{cnclazk{)wfZXBYPA4a+_H~bb+2WVTEM+a`o zzldPL?jaZs77^bg?DykMJ%*dEHv$Y)xk-KXQM!<9mNLaIPgM-DO{__@s1Lj^CR0d%)lZjJ@;_mJB>!kAo z@>*ULVV8cW*cR5;GrKA`aPH2l_*liOeCSE;3#gUQ$8A3$U{X|&D9P`E60uv5Te6%xq1>k`m|a|z2Zy5gAjmyoQXM?PR_%+J zs}O9hsXR&*sRlJ$&$`l| z#`QuAK-id{PuOJe}XPzktCtVk9y4UUNAp4)S#k=9<({yq`U;S3 znF_0cGpv^<%WAoihM&?Y+wQD~vsEu2ZWF3?tA(0m2_?MHKv|GJkqe>p+wBGv-;7Hw zlFsP3*{dOX`@XMFbod~IXt4xh4Vk*{X&vN@Lt$m~A8GDM&js=v^2I{y@I(dq3FZ6D zx&(>ygbXL80>3xw2K~;b>JbTMo0#p^moAIE@%h*n>3o?b+FUJeA+IMm*E>W435~=INeEcsXl8N0JZ|We`{xungWn&+_6xzg5oIs5wzT3 zCfu+;@=VL(Gx1_6Y*P9_(z95V-jt-uu0%bpUZ_WLjX8GVME#tUSwX9abej|z4iI;o zHtPIbI;CziB$84VV#fBIaP4JS+zn;FdVkgg{HS3LP;F*h;W3fd6}hqYN0u&vmh3{W z+u7XEO23BHx%2xo z_emVKGfGI<)>4H&)-D~{bG`o@iqXp)LAdY)tW-pXxHbA^etIxeW$fc{o$@T1vhmpe<_DF#~zpE73nx23o= zzF(1$H&RuH#9#{yCx;>+Wbm+bzC*=L=KWXPW_Ur%4b+U(#|%?hJ;2wAoW|>XW)0!? z5f4m~FPEFd<(*@>*~1TTp_8c3UO)rHcc}J6tDG6#ZmmUJ%k0F5ORnYtMm~$-uEzUX zsbUXKpFL%o_ISp{4iQOjSM!O(uVF^5_R-`N7<4v7HP{ff7iPDvq_{+E+1;PpI=1=` z%s2YuR-*D_({$)~-BZr}-eQx#9M!qOJ+ppO`;Kx_ULm07U8Jk;#5&0*JOoz7D}+2l zW%dPz(gaThL5b53!X8DVIC`5lf3F$M3>yfrKH5+&Fw{yn;ZoJ9M{{rjGrUNZkXnvc zv}47`xos%pY`2cm%;d*VZ-QuwM z>&ku_--P#^aw9&*%YH18b898;HO7q+7jkj@?E%M#q&uum%MXH@{ck&ZTeU<&4mv}2 z+Q)Mx+o;E~+ib_O+P2#v-?sUT>;eS%V3`+J@#<}n|Z|X zxeKQq9W`USae`tJg5@;?&8g4FOq=JI6-F)`dy`74 zP1DD#&*|)1qxF9+u9IkzBQ1aOdizpZ{##={SAVmZDeMx`anUIs$)-)rv|inI@f7;& zJ^l4+6tly4f(hM8Ywae@u+x$cZJ_R`G&uN?>u%D`m~ZyW;tf7<|6!Z++edIU00xc(Fh&4IXUef<-gcvf z9z|JdZO*ryvA(AfKU<4n#W$$^;|C1@8h*W>gVlcl#S_$H)I4+2cxyHG zStcy4mndOoly82F;YW{^J(*Ra^aZ12h{bO*TATy7cg1!$Prmah@V0IoG!3&p{-6m8 z)+6O(Ln8zBOM*JXv&FWagWkgB=@e>Jjb#Cje+l7`MXv*rw7_D93$(Fg&F_&7amRn5 zB_n30V?#JQ`8&V9F}M5+yxyA|iDQ)Oy&=572XS!;QQOng)=dF z!C|Tjk(~o$_{Uh7Ba^Py?$WIZ)ZY`~ER1*2t}sj$iEIBxE?O?%LZQ>MVoK1pSLX={ z8)uby$M~9NzR`?iqYGkt3}ETGsO7r^cufZGCCail@9U|zeIF0^HTlBmle~ zW`mh^@49eR*rcSRzs4BPNwx?G%+0YzH_?OHX(0f{LcBm^BjP-(lz3=zeDaoFESofc zD(NuuoNX8|u6|eCUoNF_$G&v3Pk2i3FJ*L;7+UW~l%AEf&!LsKj=bF4Kwj-RygV!s@EFLlbX2Eay#X;@3A*MCtOvGdfqw}!EZwIwDA>f z@XnyZAln+-4NhRez!R;~4#mn1f7|=u{#z`N&@`$k**nEAe}wV2r*8ML&!bACW9#oU)c%XGcI z)Wk|-Fsr&ZcS``&yJ|rGVzMs04xcWojwbEZ$LcS-1aElK%ioUA)OdU2_5)!!>=M$&3rD_*YIIJ&9+u)%a!-o)3_JjC6){Al~yNK#=( zJ|6c{ZzViBdD+f-fS{}*%Z6^fkRkjJW<0Aw`v(VtO?u7NQX)-cJBhsP;zAcau0XH6 zX0kF_%u~#@v(yTb8(vmHEm|@Bs9Jrnf*&QXl|%~88_O&F(hdW%!6w(gub|cD^iYHF zl-!2#vnd}rT(&2dQ9CmGSOi2P&xJ(7@%smhu-L+1czh1|^t2Wkcri8L98Z00GFQ=7 z!t)uZ$803nqVBucmX1U5zB@d-!?)ykr1ky2hyqslU{UZ1kr{;+9Ze{VgGJyvHsX3K zJSY1p^{0IR>t7-XSHK08??~-$t!&v95_;L?4BQRiWmBPZ+PCNj$ZNF6!eVG1KN;}` z$1F({L%vL8};#0iVMf5ISV6&m39XGw4LK>f!T7n@p7KC{mGF0n%|>yI)Mk|Re? zHR(2(geuTHg`zqq@n0mdwVt5@MCp7pfz>~RK9%v^anua!R3?Trnpi5F5OycdF7 zri)0LF!AmRYJ?B?J2<4#IO=sZe#qu@Wj3-TcYwC1Qh&Gor5Ui?f>{XiO~4anV>5Sk zEK;uP#NHQ+vYj5w$P@*Wb}APBgS@ zEGz{v@EU*o3QrByG(RbVMs>qK4u|0YEpvM2^-a&?=W_h205y5Nn6iTIn|RKlyCD(Y zix+gqM?SEoJ0sb;Ve8X~`01C}N0XR%a^WhR+PV9yKm^dxV3iwN?*?z9N`L`VtO>_- zm^H_HS^}*xyKs)8+U2kEx0?46M_a3+}$Nu6C4_EEI0%YkO09# zu;9{oaCawY;|{|)=lN&OT-8NY?Pt~2+DqQ=6=5=JkKaW=0i)ydoaLOk%6rgi1>Zxn zW|dE6^j#4D!vf$t6AhvZ?Uss&857Efp z=vNERIM#y_JyVavZ!$i~X9P{q6gl1xk3<1TN}Vo$CD}c)$4UtC$v6pIRPQobizhy_ z+f1vVgp(C@mU(p%@<<$#WBMJYa~V^TTkDVEU+RNGMQ&jN0T|hl4b5fF0vPqw%vv$b6hka*X@q0a z^io)Qm3e1K?!`0=mj_9HW`d2G1x~ODBD%<}4cb$FDl>zUoMO8)5^BPoexG_$o{`>S z!SGfQb%sJDI#G6LvTfD>aWbQZ>$bdB5#{JsH4_u8zat-#`$K*Y(~K;D6z_Y@8oNy{ zKV8w{<-iv{1~{LL$YjE9dG!zR5APN>$vTso9hqrFT^7Bz?k-UuEuU=LO1TKhVDTQm zzMyDY=aHEO^=(6G3?=mK8e= zYcaOvdYH(DA7Y+Wf1)qr@6w>#vwqjE?ucFH>vyR*+AmsU@0bKf*KDs)sVw6ALRLRq zEXrYyY!B+Jz(v;hHI4kqJxF7NLJ&Xoz(rR1@620Y*-%96VRrW4_gxCHDS*05jV>-% zT@SqF({C$mw%A#Fl+3L;lSN?nVL$L5I9^=>WLyzO;5&R6-zL~Nx7;t_X60tNpJI!O z+-)GI?Q=YITUBuj(Pb~1l~-d{pn)3!dDcR5*1NaAK-C>zTpU@^pb2LD9_=p2gTbiw z7~wIrA=B;F#bi`fWP>Uo#Dgj2uE6Eiu^h})?;Gg9^MDG`cl8We;WU6DUS+zt@(%O~*saVwWaSB9+50RZUQR*(Wl3jfn=jlF+1e-p5 zkcMxQ8e@-g<;LyL4C$aDq1LD0AwN^jXXK)B;jjeCJ*6PQA;@p6zqCwDm7Gi~JCW_u z%bYlu0CPrsxf-{Qrvw9Mia~yeyXVUa=yxj}(rkfKRqF`p(Ca4(mb5>dmgE<@ukGL_lE#TWtR5S4N)gEMc?icK-P*O7|)*?0=TRy~aEy zEhEGjyPv{$!)lLLYnR$?uF<*Pk+aZseeh&1pjRtiM(~xRFI8o);6TJIxlRK1^)s%C zgwhMhhkZV5fVrFVR^yOw8aAAoS742>;tO^^;gQm=;%~+89FK1FaHj4W&6$*G6}*WA z=~wRajMbklKvb%7Arx@5otI|2y!6RmqojOL0A!PDy!+w|KnvUo_edr;t~3#Vb9Y~U zcROtO=@XHxne@>)CiNl6?k2fYOI>$UX1>g<$JAe4PB-$91+_Y_6(^PT@*=l!RS<7f zT_k82g>Luqok3V9GMq#y<|Fpulw#fLD~RDQ&IHn@V;izR0z#m%4tNp9uMCqRFAP-^CG6fZAQO8r~TPG`LnW>2&zIYO2;bZXxXfK&Q^H`2X6EE-bL)iC5+XG^~xoF%4Vc$`LGq@C$EDH z_-cin7D3&9zGBqwvWAW)s)N;yKCvao-c2?jzo7C?l%7PkY0y;fw`(~U?>MQh`7KiV z0DX1GNi3GfkF^^zD#QkOYA7-q&_#d-QZ)vP_ocPpIQ)*9<)&Z8hguNQSavE>51PH@Od#~%XM8@S}gp!g{M5L)~JRhHy+MP_fvBf$u zHOG%4;?&{YY+5Er+@2dm^%klr=Bglrb=J84%89QpYL_E&pxc$3mE3H%_UhDH92=5b zB8tXWOgVcX&ay!_*^pZH8%5=Af^Gykjv;E@U~muX7Rx+U9gxSQVL2Xb)kIDT6v2|N zPb_M;{->Rz3_Aty>zWk(QHU)e)gG<8y?4jIsbt@8#zK5OCemd9sY%Kyewnm&i^z~Q2hRc zOC)DqHHTWlON%xtlMv}IJtye?Kt`iY+3 zUEh77dzPj_m$? zbe}nsz~2~G`EQ$nO!~D-{F+%sg?wLB*VT1cHH0Y;uj6_(-)~Pb!7Yz|vRX(=NGyFM zrd9kwA>tA_N+~Fy`<~>%%Z8hv-O~p8uhN@pYn+h$LVrvvUD%O2>V&tzg_!o`&`Pt( zbaRc=woF<;Nm5DxC8veL_v0Wv+sJqZSCLA@WQh{1FS2$w z!kI`9$Ncs}>5^eze(S?DC>fnvuw~b&_&HXkcFliTQ(<9z6x+A{cW!eMKey)PBHkbX zF32Ne?-TkOZ)RMIPg$KH@oq<|`W z+;+Q57T`K6?c+fOEW9e+F1Zzm^PVD~^rfpNl5p5xvh4Ba?BaFJ_QhdU)5z}(wXS;< zb@R^8>gR7yPgbyl^52mw|G2~;N1N8|{5C2is-^wOO*Hi3aujNu4TKt@G9SpzN`Mg( zsZB+H-e?bm_f`f8{8*=nT;wsSg3_zT9|$eW2j9g#EUmqqJfFcqCLKX`T<-mki(ZLFz1mmp|^<4#G@&NLcQpH61Zyq zD1-#R>=na$KVR){%(^aE#BuZrzQbCoH1KrB*59ItB`2r=O;?sqjgqXDxq*m^oZel$ z($I?AaYVrQG`mtx=wEyTPL(dQTcwY1MBZcHj&SGNhz-00^So*HKYjVx)mB)U#oL8O zsHN=$&<$g=@?ljO!NQ2ZkrOR;V9aqMt<%-EHIMvrJ@AGnQzlhEH-y#m-2gBQg#C0! zW?I8bM{0acT{4)hqdL@oAXK3q-5-XXqR^Qs;hV=7y-~M{OI*8^X3ba3&B@F2R;B6G z4&*Ij5}eD<$C|*tFQkUI8BVZ#^(&v<`jmm_07Kq|2d$SOrX&Y7{IdgcD9$T8LtH=1 z;i=^c8d{4mnkz9^X#n0qC!vF)@*r{0&a#v80-2%MSiqQedazJs2I)F}p!Utao-#&b z$MdLLMDaQ4Ky0(uOm>JLMq^HW_r1|yQpEF~R($;<&2)3)4I68zoDz7nlp|T8{Z=`N zZJ=^A*eF(?2jj76`{i=oY?(M>oPcJ=IE>OuMv952i7%filP>J=spFwIrz`?*{-fr5 z;E#P`ad?bGWeW*pnD@3XE+f-DF@koZAv-nJ$FlOsp5c0J7rzE!4Q2_{IHl3G_4w zr2!9jQQk-K`;X4dW%q{#euM~gkGxM(tY7l%iN z7O#o}xPQCN&bOLzPNV+}l_5Hsa$+|PlN0cLMj|>W9@gTM4?{>K=$)K8n9oxZa`?BO zZoE~YKUy}Hnat|kMSFmrj?MO#**&$i9`%=!EErGg@<#@1fk zA7Be;NzHLRztl0E*4YY&I#=hWkZ|?^w0zzFgf3? z=a(!6^)tu!f!o)^$H6z6!Q0P|oyfb2%mH{nep7dr64RE^7(G}ByY7IhcH>&oo7;`KfQ@5y&xrQp#G}o%M!uafLXlk8aCP2=T&0XjbMy@B_qRYj;`q^1q>}+- zfXC!^fheX~irG4CLf5X3x4j?rOl&L`AW$3IZ-3~EY_O5n?;rR@rC3dI{+2og%49y@ zrf4p^z1-U6@W?USY39F?o|rvOtNT1w9n29LRB~S_Z!MUA8P1!Hw?pFho&X2E9-`wi z*Fv+ac&Ton-etLl&HEXCiudQ0IFz-Cp2$E&4cv$c{1{0JwXX+R4KbY^o;SSg{>@)2 zew(vHh`wZKf)|I!X;!=3x;-ZO5>^C=TzaN(CkaK}VvVI}B?u*IL1xdC(5ubvq7jF< z@RN48Oi~Efon8`T{;Hbvlf=yW;)YJ&Gd^NyzXmjB3-z4tdweC=OzuVt-xf=^u23re zQT+fwtDQ7t1+R(9=NWaO1sQy7C1*|$EC10J01PQ0Pvx2IqCT9Re>5llvsabh6fAvf z7Y?9L|M>0J0$WL?OCQ?>X*f3Q>;t)%-^-te;GL6~JI$YS;DimSN{+R=>(P^1!1H#{ za|*3WK`~TPxw)>b7!KylQfB2 zz(!w&+DR{^8rU?m5rc$_DT(ilhDXd(_&ugTYqB>cGN$*_Wya00(@XM%R@t?S4f@mc ztif!XL6O7#^+JH@Sts+H&d>@F!tc$FqZ(ytCn+_qi@lXcm))b3%oHjng(? zB{m3$>N^#(MklKTh)DOlcYoe*l{E<%ip(ic3#d+_&{L&<+JssZ*s6ai1C(HbvG?cZ zNcEE1JlYl+POMy?A5l-vlxaO&7QNNbpUav>R`2GcDQ$7X5fXR>Q4H_z4i4R#6fbOZ z$|C#zLL4oy)DTfIHXT-5f9#ayABo8zsAy{XHU{q5Uev37iNNGpFRsH4g?R6rg)9&= z`%R;s5$K|4T%yN+%tTFuoJZ9FdLRx3`aexu=7G`|kyn_`%Av4Dv`Ta<%ZU#&wiwor zm#3%(x_X21VZMbXOFk?EkL&GbIob$ThIL*sd}Jv4iv}AIoVc=;NB(iRNn|{qq zd(vO8A!>?nhzdP6Loc7*2&-Ixrh&G374o-%TKiyepH@mquS|?Z2(+_iIHP*~`F>0p z%1%hywaN=amB;2Z$@7SIU=q(zo;=Vak0`0d!&GHPdaKTEEUWGMbee?&|0?ERmAVz* zP1`(pMJvZwJG@h(w&6z?uzy}rB|OZvfy(4=`%BI=!?AF19BJh}HX#p%t{JJreQX-d z*KQfvO*7)+8Ou)3-7$ey1%#S*rt{l?bbtCi-HvJ1u@me}23P ziw4MT4oUh4{WTHf^SQ?e#$^p1cV~Jr2@z3pD~~*7c0{oV96@}tke;VXVE$Tr+gvLw zh~|zc3)nyj3W>jb}DB&dzkcPy>{Yq=Q zt#P~C*A)EEW&_53+*Qehx)XAbaOFswILig5i5)rQXE1z))!7rYRQ^0GhJ3A-3Y|0Y zjldY>AResYvP)}4!0ehd94e%%R0q1ETOgH=fZEu(r%gSKwo z(cOI*fT#~y${?-j7cT^({Jk63220iB(!SOI>V)jUM+=xJ*K6aMv+k$`rD6+8`CMZBvELCHROto-pSZ!^i8oog_A8*YuZq zY4~A$l9<^Y3(Pg9grWPr#u~+3vxutI=GT2JBH0)_(t(#EfebR66p(+z1gf*3NLI4E zblS!v?XBRI@ddzRrLW7Lmrken$FY7BbZ~GMNx2>;hc*++SXh0>EJq@@I~|o=TYHO| z7X;Sir6p1Db%-?U93hsPad;S;!G$r}$N2TK0fK694}k0yDeaY}DaEHoBTf+j%k|(n*v4zw-!q?A%Dgn+mb=LPP`t`PkRO{Erw6fe4kIV zyTH7ze%|kVslE_a;CxaFIZt3$j)uL!rHmIkeIT2?!pXAdX&cL75pUH{S3xxw`vE`{ zgX-Tej*m2S`kdgX&4) zg#;{+Ib9*5nt%UO>F3sc=Dy=5Lo6x2r$`kUk9KCqJqe6FQED4e7+eCZ9%KnqE$Z8< zc%;bJP)T0oK|9Xhso7MO5nJVAv{KqkT? zOwRxMgfGmvI&&k!5}j$W$n{IUmtJAZ4BiBVr>>rTII92iBuN84Bh!Q)=Rd67zuIr7 ziC8#`1dlUhR4PScKH+PHE2;Md)W>MO(Bm(kt++M+dbku;l`f3DB+5(AExK`{Td*zwXcyUXJvUp+YCnu5#cQz#@e8_2Q&&% zpZXXgKB36NB+%p;^U_gpYxa)uI$m-QS1TGH7G9p1Hu$un)7P_-^Qy|jMpt-=fM?#A zT_NwKZbk%7T=_^U^G6R#4Fh>|UVpKDtF#?W^i;5ntT3h{Ij)6nS-~3XNv0dj{?ETH zxK&N*frKLw>PKwo&d;`MRG!G3lo+-UcPaL2*7W8*o0yTIp-;}))USe`58&xC(sSP% zsT;%Z8;@Ke346QFftE$FIHHq0^g`dwnNO>Yh@N zA-nMO-Yu8(Xe8Vlu`+Pe?EcnoQ*7U@)2aK`vQ;V2kTv2$++Yr)EKu_ST%&y>SrNtV zc$C(CJ|<16Va2$NLGP9+*QhjzG!PcWFiL~jEfnPO;-{0T7zZYt7yN1g4j9lL;NHnIcjkU#Sg&qkxPaO@a+`b_@ zH>YJQ>tktPab(dC%<^BE*z_(gRWS;gZ#rm$awfX+pW%Q($j-`Pc)lkU8lRj$ z8N8LiQvIzI7gS}lcgFEJLctO1sRVbc>PsK0~OyqcctCucl^>8 z$vfiJSiEhO&^3!zeTG#TF!@F2^jSYWF|ZWB9345FR;~=FfiI4MTtYR3R~Qatc}>_` zcPzC-J1lx5Z!ese_dP=z?kgkLAF6&R6d54B@4&C#n&o4rqk!DB74Oyl0?qX==2{7_ zfMeTdAdb^`VVGIHu-7OL%7*kT$)XElxRIvca~a{O*5KkMWd3~WV&i~pqIsV7xUnS% ze=!Rw9~X|=K=CJOz8SpFEI{;DSKrc1zr#xtWkpxeg*g`(b2)mgMUeE+j6p z`)99gM-v?Otlv!NlghUqRqhBte08~3bVj5?-WUk)mg6{d%ysNKWUpME*oE-6-setx z!wVb4+HJnxd}zJ#&z;v(_W~T^ zH^oK8jDNZ)`awBM^ilrfI)xMW?a+qGI-=-om5#(AQR{khm%R0S)@fcES;~{(OMqWA zI8Eb}B*F5#%t;uoul;j%%QnN3?%9&r%cA$uWjwMHeVECoaZh}?lHJ;<@NC4?L>i-svajKx2zS!>YpB;@(L&BtsEmCG z2V$Qx-K(0>1Z~*N30H(#@Tsk&7w!a+lav2QXr7M3u$-{N9SP9 zc1)X9lq?u10L~bvQN)}90{Dy$37@QYkpnQ~rdPV8%6aDft!uP9nzC7STFFe66_6qu zc%e^*F5&Nkz+5}L)(5b~$iK;481EHXvYFkOBhj`I;y!6f!(EoDa$!>g0$P_)=g6B( z`DiT#sS`B!U!<=R5DCTF_16wq$Pd~3X0kEQCrcGmiOh`1#VI=xq$@g5xUzD<2yy2;wFK_=KZriXxhw?iwNz&g)hE=@;SR4U{7xdiuPZ_xX|j!AE8Ym$v{U;ls{NU7eOku!=5^IPrYfJ9VT2#JzAs7j}$JQ&SQUYj&TCaDHz-0*{v>Ybkf_GW+MiX$uk?2x^^ zORevE_Mux%I4OdVg|PE?=C9%5l={TM@D(GM>d{S`;gLUKisH;A=bNX6TJc9c)U}ev)#;L1m)|_PoJ;2dM^v3Xg!esp18+t>+<~=!GV`IutRyA#51i;6H84JU3uYg#@hnpFo{lT+k`9V6y;q) z1|(@-C52VheZi657d_-PXGqaNYTot9jcKENc_gG@i`_*R*b)fS0IQ~8WtvU3OlUgJDv8gP@fWmOe_j=(_Y@z^Nx80=lI<(UtnD3ipu#f=;yeBMx@O z13Hv*pUgNQ#=t1+qNJM#TLxTMO@dQPi-Q=aNFpgO*607up2SrBmZw`vjQ|8 zR9Tgc7}ZzLY|{R^lLOC(rBGt@7^(*OK&Hw;FsWF4BK};mA+Vy1={n<|gjc9X)Exqh zMN6ew9vA}RO5bY^Pi4Ip!-z-9m2ePXd*#f-gN6jgMs&6+l}H)x8kK2& zqvixEg1;@)#QT<-P*P(v9DcSfz5a+9b*QJwS1l4*0b*}`d~@7?SSD5PP?tdTIl&@S zD10la_NzP*@K9gACcP{=;f&WDuKx;Ggdub86M;3%HrCb!w=CBT@V(tzMn%wQ36b_A zGK?Jq5lxxw5xX27Gc{M#MU35Gp+Aavh~?6Gc0)k+m8o#VL}CJR$n3cAZDc|KztbZxr%t4=v@x^q<&?HQ%Z~pcBM=Ht0EfEAaP# zeTI{#39%CLB^}5ESj$8zIpe4pl3hYKmi0@3DvR0T98di!8+M%1qRIAkF&>vf)4ayr zOookjwwoRFeeb4!13fOdjas5_%@9coGW~Q%LW+v8hW5HgfcwKK8DkClekUNC+4s5g zPS|Z-h0&hcr?kNJd4t)&@OMyKWNRpc#Uo?DYFdS}-vFRmsL()iYzuMrGj+7-(;FOY#;IbHbXmJdU6ZzS`1;|a9g@>)`O`K zwB?l2C=4smT}~Go(o*Y*B{+|2$xJ`_E`LfPS4F2(n*&Su?Ayx;m;DQMa~_m=8+Co} z7=KvlZUaUO6XVB#DZ+L)aGVQq+)*gnk77BvP>7Zdn2UwVi0e%GHGzBRe@uC3u#Wkq zyRqG#uelm#i@ql03JxBindXQ%3oBx6X(+RgDg_^q0c>i>s1qsSE?*Zq<+fM`3T`gf z^_Gv<+Eu(>g^!)f0vFyvD?Zw`{Jg6l6gcN-Ma3gZM%fuo6Dj7k`pTq8`Ur9RLZ(UL z)yv)mPN@2`>$o*1U^nJrVkk%eGPR4ctgQy->FHQ~WiI|F9znH9ZTn*X;t6o1cC=Kw z+o|ZAn-M<5eFEgIljylqd?m&(VxW!elBB6@WSQeY-U*dz3MWWHmKQyWB%)gwd%LoZ zD*4AETy3sHkLM82De0%;7ZFlU(<1v{LBb$F#Gl7jWF?h#1wTUG`5!u0Lwet>@0_1! zdOF!04b+BiL~EV;eNs5U*WM`2mu^KG?Z>Ol@=H{gI)pKa>)NQ~eJZlUF?eQMJKM3T z;>JG=(h7;oD3L^V$oQgbZ;szId`+-bI5=7eeP26VG|;s+_uPr0z1t|!+tb4 zPc5sH5q0s0q$TtBKD6K8yA#>kBY(rc`PB!##t!-9kWKQ|O!N#ode0;YnpRi4d1d6D zayM@?-1PPY^L62G3ulzxJ7Alba-VO}xGeAwkA4+fvaz-v$mSi8x_FEewonu8o*;Iq zip-n<-%%sux&uXATU`pM#kbiHDGM>vNG4?-=s z57KSbN>?Np`7haRR8R)7ryYSgCqx9HCpFcPRHGUEPdkGF5ot_KsU_3#aQ#-V+mibO z7_*7bZ-}_@?GS$yw)=cHOBcp4XPy+w=gkl?X7tS+b9CV-Pea`q6wU>pzeY#960{Su z3$fF~nY|k!#-~&2eh|Z!mk7ewPV}G&BuIA!qGD%l8P(pGawz5Ed^vJKd;u~lQ)s4s z3iAjFS2a%+G4}o0l>y$4?Y^RB@Dk2Q@ohw*-|8~ND{ zWtwBRU60$&{g>q2{v|xGrz*(MC2-9y5?AAsyoDVe4Qp)CcqCU#7gEERq&bzfw)U-i z@%237SR%81J&`5_$m0upbw~TD>949R{DO!msX)m71__XkK?cs8-qv0jztByu@#kSF zKgi8JyRyxG1m_NiRUt=Tp!WLHt*jrrQ!0S9_5@MhJa(MlFqZ$FWVeVU@}cc7w12MU z?r4ai$UNvqNFDmA22cayg zNz3M4RX0ZST3!QSZh{&g+3f*e{M|rR@M?fSk0_zYy($8I&Typkod-gme zDIVQSH=HwT;bhMmjJ%%&;u!eI=s7>K%!=q(_RaBy@4_D-M+}0sw>)!=B?o4uU@1ST zAVy+7v-OOAyZw!If~WBY_Z*!cESUKV;aIH=aR4a6n&WC|?pZ*p)3rt@9;95@ZE+q> zNdM*jHenNzgpIeeN2EoC@4o);k_+8ias@@{9;X9|W_(Nycb6c4ecqV`)h2H18^#T_ zh((x-79erm2{=w6>|}AwpO}}>rL))z>}iqv!vxXY1x8RvW9HSE_PA~EK@Muo!mN^l zxf3Zb>f9&f0++1b0a^nlHWf*AW3&;+Ue;44Y}Gbz{qIr!62wf%6)q7m9etWdqx@G$ z{u}~wmqlceB&s&O9!m51DkwVmcrT@Zs5$)}{)ML#PbtoH_*0lYUsYU#8^{- zNj>{QjVeKG<2u&F;j{L|HW#o4GzNLEf727Y1cuhDWSC|-B5t-$v`*BXkjCHT#ICbM za~ZZOCbPtk~8&Yjvx4_AxzOnI7zeNP%sR_S!i0aK&Pd<_ zC2Mfv-CmH%As4_pHoW`AS5^NnSP9+8!2`d(Onek${i4JXB7ZJ))cKLHQoq0QP=yWNBfp3U zq%dpa$F8@6@?BD%rOi81TVdw!PihqSXx3Os9f&wlI7k(Q(R=n6PR(!GHg6#N9w8iY z6lLPIb~v_5NV;NSOVPfowf=bi58s3Ld61+|LnUG{$1YJdw|%I0;#6t$ciArE;biR= z)TL&eo0oAA`~mPk#w#SsXCM#87I zr}(_>M1$^RJ))oKL$Z(ww?Y>ZX#{jJ17b%xPTBYR$0CF8)n293pJVagJ}~X(^r6m| zM-QQ#4usRAMkM;fG%UO193ZQ=Pt2=Ev>!A12u(w5GNg1zEY?=r)DbQMqQ=!pIl4=N zKJ8JH=Kqyt9N}=cu-JO{gbrux8^LHoaV-%1Hv6OGw8Xi~>JbPPy(X^)6c&LI0!{K* z`9YmMJ&;;=&9w`i8vFUMYOxaTR;=kRhUid0Yp1>p<$BZ^wTW2tPDaOVR{TSCqU0vU zTYc*U=UP2vHXeOrKR{Ka1DmL6TT@#Z59dBxK6(V_LH!z?loWw3>Ekq#3lZi(A)=}^ z{@F%%Dl?)o;S;J7G{4!lm4-9`ucvbhI}mx-+`JPntZ%9(;d$I{eROgCu!MTZ&~KEC zh7=pgdrJG5I+6vp{VyR%KX{t;+>&a`OA;*dbp52jmM7uXem%Z}#c-L6^0Qd2kdCnu z{>P6^Wh72S`nJ|nXtk*2N&iQncZhGrr%%gQ+Pv^E>F4PwC8GKb3q-d< zQf%#UUav7@-Z`k4@~TotkXGdOQiSk^CRv_uo*zpHYubj28GFgjXyaOgINRP{9LG~w zV@(x;mHB8k4iz~eChj4Z;Lx&GWHX;BQ@}Q6?zr-a~R@(p0#)RkScLleZ8lEHJ%=>3w*L&w8FMHfYHt2QlsH-sl>J^Q(2^!(Agl;jjqRRkq?e6A^0%9cTZJP`nZywnya5fv)Y|R-C4_NtJ7-^qzM`s;S z@Rwm7#R;l|=yK?#I@6!0c1sfT2<P@Zjb$C|s?HC{Bso4*9t{mLqMDGVes_A9HTHcgiQvbU{XJ1w`%q+HOW@8glNj-x zH{IILsb}GP*<7Yw0{x37S^iamuI6tCpC9L0^lmOpP6|_fTmvwunZ7;qZKWccCDEhj zmq;GJlJ_~Z-xwH%>?%tHV&XFgGk{Mi3z5BqQ&(amUQuO$0p+9&pqdNYqM3PioV+RS?}TaYu?q-V^qbDFy_wMR z=9D(G9xnm(F;A*=OfkNfX$knnBO+h8{#w6Z(^4DgVL%p?(FRg@Puh`w;kOBc^J(5r zd(_Wcm%K!^EHlfSxNCR@ z`JW=1&wX6+um0EeHv2sfqr-HC&7euTKPnUlh1VfTc>sVBpc2xbfiL|(2Ov7C6fmhf19FR;fA#6~@eXQ^Kh|cD*-VyNV@xVy5Z2`C7 zERx_kSJ{sUHMjZzl@IEqr%?5+1~S&4zR~;2OK}?>Q&|;ooanSDkg<=OE@_bng}YiM zh_9<)UI`7)O2H8Q>i2RRy8My3uS4W6{&8IP0QUh|m4Ky(iDnjb_003X3y=spn7DbHlXj+@7=^f&t zJ&E-qK%M!{-%l#WSVTx~>);uDusB*yseDb1=j5FG6n6C2a*7@*z?aw}%Yue92Te|$ z2dr(-GsU0k4jB+V@+np{aS|206i8@Z97~wl2;dT~+Q8+NTJyiBj^X1;7s-otsz&(! zBtL-oAifq}L7A5UerGpYXnzA5$cK>~AI&RK{6@$;Wh~o5FzU!PHzi^H)Yel80g{Zu zYa0(b{liUn&)>Lt4!k=A8ls^^WKVoN78*&zdugX7;|a|rQfsKwc!$ZY?xQVMRM6@_ z)%N^szx<~8v7h9w>MPt^>RJM5&ZyP{%TDlu&kG_(&=#guBgyXe-xiJaD$wx2nIgu} zVGr@IjyI5OsB%i#BUB{d`L3#_&1*6JY+QMtwbFtytQZJCul1 zr1Kt*O3K$OInH`$<^-usS-s=lb7SEDtY)bDp^{1S@W*&cg@YI#)orrr>c`9(*&9!# zZ_<@C>^P-5iK)+BM<2-Bx@m3Euw}m4!vp*jNPbpc^sJezO5GKje2_o@K3OE3x3Ee% zy1f_%v))h+#vk-~TP7=5hsTtdv>p#$v7ge=?$A#p5MMT^oIY+5TI<-JYf|{$BE`Kv z(tYzK%1pD?OJT=?Lj(KeRwWZ*dy~R(hpi~oMXc5LCCV?4WH}jsOI6msld)~fvQ)hJ10^381rrU6v zMaZGDS6KX;E2uEh`( zK>vl-{Tuh(#;<-_`UgwA^WWUxSS1yUx$~=K(vfzz`*$*JbIS|JHuo9#-w|A?+YKka zS^6#rsHTVX41aWOE}p>FfW0{eX}{j5npZ>%+t#W3c~#|o%| zc{6B@^0b-1K^J$=WUZZGK{!kD`5&9}PjkAY%_n`OJ5;?{xJJqr~XeF8>k3e*bg{>__j`7cnX7EA!Sl-k=XYt49_%vZu)RKS|jj zxyNYH;>p_&6SsH%d{)KN>IXTH1Kx5*)M|@7;>t4EwIWVUw?{jYWJ{$yC3Ep5uzHGRzfY)6^H$<}{GGFMFrXN4Icu?>>8mcQ6rgS> zdzAn7=IxsG6+7Nttu3pub{WsV51)#ntm#rDYtOsyaHk8*oS7val}`avjNV1V%+1mb zkW6$Mj z@b;DskKHs;l7*u@xJV=BsypxlZD32jY0gpRePLypG_ukF11k06A_F}==#qT<@^N)V z%xqlJ;&2#HWanpB2H~M!svu*D!>gqb-eBL9sb5zOs+lw&5;tIdDz3;^c`nHGRme{| zd?Q_GsQGl08Z5Ws z*y$T+t1Rbv_Eh$_>=?^YxL1p3@&~>?_E!khaMz}nw3&y2-9gLPMTu2!Pg3UJ48HRP zTd9k{ueX37V^(TZ#z!<^O7>|b*o|8|9AFmrkSBX7bLCNr40g;~ao&#vUS1wj zy_spO+KLmX2g=1w=V>2gFu&v`r>Y&@CAi^nGTThRrS{XGyK12(IFx- z>l~ep97%4o7kbGwYJ5XOS?XJeR!?yFY+TR!{|^4&qt`9v&1>XCa^IRbb8HLt3(R93 z3C^7DCWJ7qql+T_r|I;I>0C#}$jj*QEcxH|{m0Azl&^1bq)WLc{y$6pYZ=_1Ub{@T ziH>?a|G$;J&J68R?u?mUw>luC{$ES}AFCMtt4mS33z^#VwyP<$0kpPwC7fq%v&}pV zTJgvlVFEdRX1RYi{kyE4=&wMG zWL^l|8D@ey!^cQ-T$ZwSp}NG&dj z*Q&<@Ndg=5LfZ+w%T5!ntwFwd`T{M9myYVD^JB{jgvG6;{lq+0CQJ>-wH7?6F2~^l z`TL}Erzk6xN4=pDpH{b=BtPy+r(PBLD_Cuz1I3I!HE(* zkU`uCmo_#b53wvAncUb;iYFl<4YA?C7 zv!jzO>mIQmpPtMJ8}_L$5X^}?%}CF|W-#DJYccm$R#pj_iEkqmyOcvm9m&=t zq)s+bv@0v(mXABz;y|0E4vv(3eAlIv(Dn;`DMCcG&kU$Zaexl7}K;cb`))g zYa}CBV1>p&#*TVFsXw&GS6;pX=jcZ1y-8cMhluMZQ1%@p2yZ~~{oR}Z!9^UXMAVOL z-lsJ8w|$$NIPpU8;qOH)!Ahru#(key!_a$;k1u644%}n{g>P?_cTWG=-bp@~6kem3 zM0OH{5Dc7z8TW48+oTS2`mTbXIEmGDtOuy2we9S(XPJM+D{W%H^K)Vy&Ynf1nSQpm zwM{BNSgv*_>YuHss3DA!lSSB`wb8^x@dVMOmN})(&lQJs&Mqn_j4>fe!1O;(EC$V0 zk21Q~q>PJ8q#AimLQfA|x}as#FtG7H{z&?g-}p1D9=qYih4c1V{F_8wkS;Tb9+=Wv zZ<#($AEp1WLGGab9bg}oab77P?dDopPO+FzNHm(wB_9zPLBq@iqEjSd!yjZaAKg=_ zGkVs0uj(`lwfVk)IKM<;XM@Kz02CMb&0*pCOC+v-x$TokW%LG-kWIbiSW*IZFFGOg z&8FG7mCf;WhR{Kxp$@0`b%te7h;tZE9lTQ*yL}UcjuzIA_p!Sjacy*DmeIUUd7Hbm zGl_6(cSt?eaDI;M(Do>8s+drKEsamczU4$m+v_^nc?NyK?O?_BlF$|X^yk|h(i(;D z^p#?+=46VjnKWFMqwKz)Bp$*h_wV=hCT30T5bEGD?3lQsP_IIgLfvqp-HV4ZUH3>E~t#bBS?p1o8cP5#Fu9*fR>h_)ch{POOcb|Bt zNp_N-Ww*HKPKW_6A@)8oJ&>7+p4XRVD~lY?l@~3I1+8xgH|)l`4On!Aa4-g#@!Evd-O%!K9@T`snF0rtH198(aQL$6+b*UtAizFDYcU~sI+safCG1Y0^+MBr)^3~5DFkC7O zw^)oJ;h7G@7$8F-zET%)b52#Uzq6J!>7x$pyqHT2qKfz>qHWHiw#s3^@gWiw)x0p_ zgKO_)z1yKXm*2VE=oDmsrud)(hZ8@mIwzHymTyn;A*0RfdZjS)7O|Hc;Br&>>4<3I zD!WYb(;8%Ho-NOdCQ468mDOZ)~(k~SR4w=Vq^brx`E?g$sa7gXMe_n=SU~%bFyLv4O`cJ2C?@{GM16732E2G!6B#LiWV(0DC$F6sL6#H*A9m` zic4>pB5*>vPUVv{l^F)S2oKBLy-uT76*eg{bVDjtHj^YB^JQb zipY7EKsXzgM4-~Ym^)kfW>xP=cQ$DxgbOE2PQ#+VJq~&@^t!|MJ}}Ieyi}QUTT|B@ z>+TI~$*@GmJ?sSzPppt+tunybX$Q{erxo0 zD6(p|e2BhB?5|Uh5S)oN+V6>@|772->8f3c zx`Z9Jp~!N~Aan&-t8n_6237=`jYUDO}Xt(<-JA1xEVK|;Y^JKPm-lmi_Gp~nX8RUnI^C0h=j5S#M z7sw=MRAUNXplb8`#YA84TjtKVH7frgNE`2s3>|V|>#SV+W=^28)L@c_J|STqB>vXn5uyJWfb+(V8rD%h z(ZTn0@Mg>VzQMW51$u!|8ZZBw1USdzsL+AJbFr8R9Ek_uTl{sp+%cFFv|08&Zs5l^ z_f^Ez#6(kLgDZ7?Pyuj?9z_zRH@@T&8vH3qk%->2)syKoEO^Vfl3W8GJD*ED+8J#X zfRBHX7)rsjDXK5504BIO7BmV%-0V^lmYMC{lzcc{e*vMUW)>BGer8{!N-Cl)az2}+ znyIyQPABdqItuz^xA<-7-myO9nSZ&Fif=A)!}SfmB)P>Ba-s&rc6Lc!z0_ zCZFKKieG5NPj_tOpot|7MwM7Z8XTWYos4KwF?){h9N%2XxY=_&D%QYa#SBW| z0Hzr7WD1e9IP9roK@XDFF-?8cYkbczRk!D1w2EAji&9$%suTHW*xjspE5 z|2_p9wn0g}+<-=nHgoLb#R6|loFLlErX0BQvH+<_PFN8uOk&x6Voi8Cfd~JXLwtY( z-MBB4*Z07z7be#FA@ZF#qEOcGn`HGlM=x5s?Taqa+xO`;H0}#0s|T$#dJQ@;Od(K# z^3BSb_d#GWIr;-H1`SN&%}~dBRw^B^KJr@{1lFLMJi-M=#xiNWNt>g^+MOyvpWk6o zpp94|S=@^>-FdjRf{uq9-BQzZuU#$6$^zjB%=DAEDfk0(0EGit%LUek7dq(ih{A7H zcGR6#2i$ZvT`Th2mOXYbM&sa6Yj-i`V}}=n@;S7_M^c$#3Gg!V!bK6IZD-4DWNRpK6txHR0|P6Xqoy2Z=%C zjH_eGYMN@(X%kODF*?pSMc;^pB28uWHd}6D4uF^_uqBzJ!^o&h=-=?SL zTH27ks+{;_i_Njp=+d<>54HL&Bt}5>v;FpnL!4;9%;>l1KQ0A3<=;Lpa#LhQJaMY^ zq%jSShR2NCh}jn$Tr_#vk#335F;mr&0h?$M#ZY#mS)7az?8mis2cFn|KVK5yA~qta zvF#BTwQJZJe@_H*2OtXrb0&tYeYOS20eBmj>r0CDI*j;1fMj;BQUen~n)-GR8u=s+ zrdeF{SBrK+vxev#?eU*hRi$x;RY^_u8Il(n=@R_rp=W*kj(|k(8>PN7_ z$(*@yo)EbptvZqUQbM<>o;zM7wvNYgp_iVKouQ1%`wx;-{8-fD`yp1)pBwT7i)mUT zSn@cS=w}xfp95;erWg4JUJ+moGA0(>1r*lLm%BbA`xg4Ew%zpV{Nnm>r+*4zoCfY9 ztWZsZFTrWuI^A6TkA-g4vFw_3qc2YXiI)V30OEMdY%d5m8}DWyhPlqxeVy;*r$Cc5 z>g4fozbe@k)X+$PgcNW@u~l-Pt*?h}Wmjc-#1Iv3?$O^KbHBf_SD2Q7L#F?thvh!F zMvvTG8ecH$?WoLazhq*7b#sK36Ve@u9Yc>`im@c*Hb^8z5VGn-t3>2Y^dl(lc;<|! zXOj*%kJZ*gZG-9bg~sSiOq@OEJmF|^k|)adChx;fcG>rZpW_$~Otm)5sor7W1iXq0 zMGPc$m~cXu32Y&rn(VGK46W^`qO#30fOe8#y!i@TUR+e?rP2rzNJBV-*NRNt1lJjaHYA`qugHj5MEsWgDbPEvqO)|;=r!VY@9}7W<2CIdM zj8OU#A)6k9@q-AJ512JA69>)dxL3%}Yws16vC%+I1sRD3QXXjyO~Ct&9A_xWnj}V$ zR-;!mPNr?|`R9;{xmmRz&s;)AVBaQ2?14Z@@B1xfB(nmR%UGmg$?iVx#&B>*k-FeSmGw`o1V;QW%+OEl*fZ zr#q)Z<%pt39)U`zQfpfDFf^`5kM=XTGA;JovljDa(qaHdUcl(;Mt`KOpLQ;c^h@l9 zKRe^U4STE>_BM|ocZE|~DteM&SSlFMDRkJu=L|}X<$2{P!)wC|+|6km*I6uZPD3(d z=e;rp5(LFHMy*9fUytu%(!O#Oc+yM<&7Ncf#E{4ic43)sgy2?I!LAmqcQ`LNk<=EAWb8zvov7 zNE~CIWyjzcF6%YvaqQfvqpcGMYWye|@a;UIFy0geUQ=n#_QH2};t)`?ZJotX$!QS} zu!PV{Gm#^UH(u~4L_Bkl%psZQ0ipA(1%=u~sYL!nUx>=Vrg2i+?zDD9wR?F?iy&|D z$}=W}g12HaNm`8q;zrkzt9^XK_48yX{T^DL%;3S_^cZJI&%D zN$~*M``d*2#5>(s{on#hT?*ODX5poEIeJwI z44T1Oi`B~NFB^R_#V|>w8o8imo+6;M^E75y&Y@23?I~nE{gn|zP5fvv6MjYJG%FK2 z3DjC4DxWRERE5UgADz%A0pPXOFpWQl9|zh)3-sB)6YDadJ}HOdO@#UwLuE`0gek3m z=aF<;{IvE}E)Y@`Y~vN9i>wXu>*gk*Fpa*_JaUVn-YVdbF<%B+df%fFoskbu;a;G` z5IwwtU)#J_jm5k8fs z%6RPKAvy^0BNDaYu*PNV^$LN&5)Ztzg2(--g#@Bp#f|WM0Kf2%T3<0|I*i5MN6*`O zbbQKo$$dwKn)N*sZ-Gp&M+kwejN&x)ki7;5)uGkz3T~gvcK@^cEooRp3e@H zerqYAzCzQ+LDX#-_|R;_)z^kDEj|d&P|}Z5#~~-~2p#zHzA_B}oHWz^_%A(LgWjYQKRrYZS-I1s# z+=;vvjk2cft+P^W_^0?+szCTFq5&`*P{lJTVeog0wi87EQ*mji&0|wh-{9O$A@h@g z)rfwxi0^(b!GBcPD;di7a7u*V{iI5vs6*SsgQxfZ1`32+a4Kk{{X*gtKZqqL)4SaV1YEjfzH3W= zU%1sqTxtIg_F!fIn}itQ?i>jGd+ooi$?vUyDe@_V|yX@*j-z^Bo|r_{sc$S;9~VkG~%Xw2sc1 zVGPIXG1Me+lWmuqsb2O<*{1#XyZiy0vYL^%$}HTZ^HjHOynF@7=L;p7ko%7N>B?i} zBA@By3aRG_hOg)U>s9~t7(xU%#vP(Bn}r|Sq<$2kr76(Ts*b&ZFJx2?CX}szbQ*qj z{8KH4twfWs@4$&R6br88eiVa>!Kd`k*XYCk_9Qp#C?LQ+>tOUFRf$>^`fge?v6IPF zQ!V5q!djDLjw}8Rf?PDO`qmXSdSmO~K#gA!;fgE0XP5J4Nk}n#ZROv3^KZx%`6d9* zjkvK{UPq^i`+#Sy?70{>49`L{ki^iO=~MK5p~#AgHMOpHk{qhJqukf8I{W)p&st;uk5PqrkCf7a}|VXiX)Y;Uq*gKG3<%|$<6q8kqc#|b=tW3_+*F9 zj8(JZS)}C3@4X57P$Zc|=ooN*`4;9dxw%SO{x&x^|5)5fsZv(vdv8@P`|ag#@d%9h zAZUnCRW&_3TWE+yQksb)yhn5uqxbcn*bYA&Aq^baPQV5Smk+%&`n5l$Fsy&^EQl)m zSMV3^?Vat9(iRqlf93fStS$IUL4si`(J^B)rhU#AC>#4?u+qf?2>awP)dv2pm9plGijn5tof(5q>iBOD|sIgPJ;o zyElmDsscCPC&xtb08lVG8VjD6NM1e+&SgIDN)Q{YfvW03`@}ErE+zxrhIZ#Z&X&bp zI{!mjTNbfp8)kv@1|{o~Z#I%(0jmLG>E)p-MuBF`z_<<)BuuiDRqxx6`o2$AZ^JBo zOz@X@R|^fDj?&12W&L@@X=NJSu2rjm*WT1`jGV5tiW>IT@D3yw(P9RSmOxz{*B>%9 z=8!tRn0{-%&wIziBNd~mKJ*KK?|Z^W^1^ZGK-m`3`EcR4(K7lnooWGJE#W^Z<`42; z4xqR`d`nVT-qtqJaw|n|W@%<7tKr^M9hmbt>Kni1_HPUE|D0<$xTiV{4(Rk%5GqE> z&$g1FTf_~?ww-e3t(#1TBq!4WXzlQ(1*Hjl2qB_%emu#=#+^lyJ4wTqOKw_aOu}bn z@_Xd#FOX7$SqBJfN%r=BPEBTY*Ep1#DiW-{_)XM*ja1ubNE}A+*zX!p*^o_(Ny_=& z=`z7EOJPw_Kia~i1E(+)ko{Gjo)wmwb)}-KX^9Yh_~A%=T@1r*vu=7?RsEEJSj4Ag z1-#nmo^Rqa{)qx-p&@#QPjRBIk z=X7E3l*a+@(|C_Hr%d9)S({I`+`Y9MdmPt(uA%`Y!Zb&YQc|f%$bkv>FkYeR`PhQZ z36)=&+-g%9f&I}{S-2xD4jQTw#Me|bg+@Q8+15xPyg{PsWrJ43Tb${l9x?Is7qRLZ z!#l_AW;&j&qZvG)7^{~c#`)R#j6p;R;vOmFOg_ZU=UOS&~~7+7Fc}#=Q~dePK9<%srh`LT=z~*@jK^ zg0vH1N97tR0v0_67*Sx(#VDxsE4#5)xxxbd#>O7O$1Y1GcL@ zR*Vlr8YN(%z|prZ#8+R;BwDO_9CA(Swx;E`+KLvStk=+snZKeEuR{hC?=Xp+$Ch8 zISOPdy1hn+ITkS~M5u#ImXISnQPk+h}gR{_awfqJk3T_`-0U2h$Yk&C% z%$ri>F9_YF)ik>iw2@V!j`izk)1W0z&gpF!39fs;3^A`Dz8}SR5r0`6$I3c7de;w5 zxT!Y`*R9|0)gx8$rBh6!ERc>)YF77fWl>%Ejyer0c2TZEFzGEG<5iL!((5-)^9r3L zepJ0e8cXYiA@O7_mT#R4)rK{m=mIg$Wo2c_@JN_Lznx5@SB{t3H6!raKjP9u{gRFI z;^oqYOB7U`)tN(+xTD_+F5mws)8evB9nCF>REav;cm;VN1KD;q+pSxfN-W3|jMlgP zXmUu+mTL&XCtT+Lp5z8Mwfuc~nj=9Aghe7d#iMCpeD~HZ3#;J$J*AP(otkzXIKEQ1 zOAIDxzaBm1R6|EjguTsv#ANfXIhn!+6P6t`d_2#swxMU^)2xQr)*h`i{(xluJq~To zc#MZZdo~8$5~9q7m)q3jo;@Lp33@jqij^nc57ED_n!-|;*YWnw#KY9w_%X|HEhl%U z*B5Su)w|0lHPqJ1D^dIhnEcxyyLubYiCd;LMU){-I}H93D?_h|4wdmgXXeV1pDTS{ zdU;eWE_V`gxp(C;;u|?`UbmL4C~F-;J3p7>?R?qH)dH{ZXzE*0skrQpYB+o-dprvW z-CDzZ4B0vixs#(Mg@G0626FJ8@zC6p&XumNCX1$xAetT8{z57nA0|r332F^%s>ixt z{mNG=@0X_z3aGg}YjU_85yt*=6%LQr$eir+P#`98e1Y)5EkEtY)7A%8F4>ZKrFaus zHt)H5^*^SW_iEHj`a#+@hP2lBBuY=mK*eQ&H8Ec3zQTOje4t(jF>JVn^# z!M@6uiDjpbMtaXe8K{JGcZsz*f?dG-mv_x!uNREL+)cLohmh#AGh72%fm`r}tT zruUDJhELHU{sxR(N$k(J9$gOLz}J2Gop}Z#mqj(hU4DD;a(i#KF-0#eFTxK~h9qhA zfg(>)@jsbsTEvCF9BVxH$g_=X0!5j-?w)UlxM^v@Un#|6MeiEoe+fEtjX|vf51=F_ z)w#;=v8ZL)`8zD%dHR99##gO2{{;-jv}RpgsNA5`tU(X)R;g@Of0oz za$GOoLlQQPr}JNPrASqh6y#q`73Z9EfkU!*aN0J1_=0~sQA!j6V)f5H-^3!CzxHh1+3$vzOv5#{0|P{^Ee(@WVPj09W*HB{+thBUoIdmuzTDVYuhgWz zzT6#2Qz`1hmrEu&qPQD1gu1XnC{1U%UJt#=bHs|`UVRDB-0k##^6)&4=9~K#CEoqz zJZviBb^uLj?zm$8x8+@~;bV{No4T{Bt(_g@!ac-QP04ou`&55T&U0J<DaLbB+oauIkH!ue?Jm*go@8dGgC3;3iOA`=HVn) zux8h+#lKtX^pz>WAUmK67k_p?T4C(Eh84cqHX*N7ssQ90)|U?4bZNUT2Xhl)(>v4Y z(PD88IG|&qE4PYwY_u|Wcb?9@jq>I*J7dzSCy-~k8Gyvla^?QE+_9qA4tjhE>8XVucf^(%4JX;=np^5W$;|&0iPV+m#?+$llZ;Y*~PwS zFzMI5D5~+(U(id|EBaML)!_UPbA0im>)N}{{7?g+8%KiE7y6_>x@5uf(6jC%rP0|T z*AWuBt(Yve_bU+dNw33Q#{8u_@B&)UHR*Q4LObII_bara*1!Pt3hD| zM~fma`a81bc++!mF?6aoH2+&i4vAOipUzaw_pem!pCrp?p7~4@dWom6mEWMr_YAY6 zExjZrCzs`!+ex<|w+1qdv9N&>zIBK7qg`uYdr5wp&Go;3;Q>%+A6-&yS-o__b5^%} zDCfyS0TM`e9pC?3m|-mpw!VTz;&at33A3Re!btDMdyPB@7^5p z@{4S_LNd04ooEw`_es?0kst6Wkxn7$<;BIn+&e!coe!RjV}Q?onZ{I32C8wbH(Nd! zaeq~iEf-<}_iN+z{yUA0dd&xun)z{I@8gVKp~>X#Ns2x=_tzia<+e{kUmIXU)QGN; zT-h*!!mzp}Hj7o~@JYZxcLkOC^$vF60+;ejjlqfRCtvDcZ{nofFP{3>20b{H?9{p5cFCSW$N|)aD4PVQgZ_KC_lBa+g^c0 zQa;ujX~OWIDh=ADJ8uIiMnFJOQ)90hNLRO5f>e$LFD;4N$N|#B|r!ot@TXe2JhwUZtmFg$8Al+8k ztZUS$*Z*pCSr-E^1(D*s(JUdyMMm5F;nC95Gk-SE!HQ#w$XIA{73k!oQXBFsdhSWN zVl-?wF3=Qzt8pZ{7G1t7YYRYXpMoGoq_#1wVAWSR0-C10r{WNhjb8GTo%R1&j; zDr)o&@%jqZ>5m9oqgCp3Nk%e^8lg~2SUC;{85N=aBPOaC{ru>>-48d+DQ;bOEY2{R zd$DmYH;m|p((0oMYY=e_n?K0!_a@TTP)Sbwf`OsETX%r_QaOEFAPO6f7mh%iSom4< z%c0=Ka7B$3@}5cSN8v1L3|?MlO0613l1ss^mT{$BdEFw*E*t znCBo*%V(b&fJbzn1(pHNM%F;AJ!iMMTWfsJ>vud&a>p_qs;18>%dGW)se;!-Pef=^ zJgO)&y|STEvkbKX+)kLa5m8TjSg&uc3->{OQBG(U80l#Rc_9DGX8#gjg}L>qAb zc(St;M=iK3IL_B)CN*eO7M^n4^A)K_O}Y;8K>v$P_j`(mFZ>_vS9k1oj9Yz%a8@@# zpB^fZnSWd_k_+OQu4TQyn45Sh^J=DJY%NP}e9OC>Lv{c1Q1g-<9E)Gtjgjtok#1?8 zMc#JSC34ZOS`bQsWf6jn{%T8DT>>}rP&dhkTZjLbjOfh^RM(-4rFl<`-2a*0{}W8a z_63gN9R^q4yKLWzUI-H0Abxz>uA$({;;+uZOP@H{Ry`frsrs=spXtEqv+m>Fs4yvP z8kzLFv(dST^u2-UaU4pg5&BM}fFD%M-95&{aS|i6#c!#x=NI|W`(E835m@t%6v5LL zeUXC2T8%}v&|B=+l;l~T!Fy-M!U!g@D_$dt2X@PBKaymlRd6DjBYCrwWk2mTSZ&Ld z=1wM_#Jus&DzCU*Bvn}OxuSM5LiE~Z3^3?)j*dResHm=+xjAb&UaQwMn05=bk1Ehv zz}gDEY4YQa?20H7pqT_g=ie+RE|h=!FFQ zXuw!2qQq3~&oHSBQhL5l?X*RTO-fGQZ?iyCZyU6ZxPBVQbXXQ?7X58V(d762{A&nC z!WAHuTP#1~>Kn>{vKW7E!o}u+qUi)4O`XavBXz`76cSZb@eU5Xzl zuw*mUc<&;8OD)ye#Xa+*jmdMG=$rbrdX;z0_@Y2)j;o!smq~@jUK5#F^K8 z_A6!&roYb@23>naa)OTq-P5NcN(zgcqz(MG@vq_LN}Kb)qm3e7V;IdQAUE60(ckTT zu?P230!a;`K`Ce0u-fk=om zeZQ=!KgU>ylXAm(bSO#yW{2-W@C4 zi*k{KT;GH|0YHmW9*HjkjMWEgCi1{>Y|Toanfh{b&)aCNie3IC9#qal)1a~N%0LRE z8Emutqt4<<`jW1(R3_Y4QP-7i(SX)(b1-(MHr#VA&2R_#0&X1qY6|lR`5eApbB=ky z3(yz;z+#anZ7_V-KG%>yTc|!!o>EHyV5VQiGSZOA)k9x3kZ-md6T<} zsPBN~_D4=PECOwg@D07T=oehNwV){|IA}FUSPCXLpz}KA(A%Ve72A7(aOi{ z^3i=hJlrU~hm3ioT~xF7z+JAcSGW3|uap0c0X zN3;Ra1uTN7Eje0Uq4PnPpebnWY;c<>t&peXcQE^zq;Vgo(G3ZnA$Tm(|lr2>@#%#lF~IkJ5MscsH!U>nJ_QyE34q%s~62C#or#@E9Zq;vmfI zbUfr&zdnP-N@L&kG$YUS0uc95cgGrME*B+&VuA>q;YjKC#j@hyV}kkvP&R{!>MWTr zio@}&$lDSo@i{+yQIn=}8zVRw_~1dzm(b8C_K#t##tt_H zwQ^s63$r+$` zZ4T2!rAi=1`4sj3;;q4Z?InRV(!}0pUiMesF?zd;jsv3{M_2;O?3lE2h3~-#*=eE! zul#-vFZPB8-tP@y;v5#;B%4ddJMX0M0S~6d$0>hq1ltalMhAmTN?) z$KM;;barLT3*ft2`yX{WZ&14j3#<Q&N)N7UK#Hy~0k4iGO0L7ONvkBl7mNElh_6Hpipbf6-d z!*0!ixgRS@X7&NOrTAqMX?vFItD49f3kUA+feg4muK%CfXzTj zJ1+WNwRZyH7L6~Grk%IAT*4si>ayug)Gg@39{}P1_fd007)Ouq@#`(s0DP0>Yq8t| zd7p3~b1>n8panuS)t+xKjlYLQr3MQ$$fr;YKV14TX(zelzoo%sQ-`dtE#GLQD0G=! zpD&-mcTt9FVgPaaHq;o?D<_Ma>y$0o*S!yFN*=q9+WE9C>t1NGiulx(803pJ>max! z*AF2)g_wUOq48QV(9!EVQ`0IT86^Tf96sv6|#Fo4Kgh zaj%5S6j0(h<8WnR%>W8%!oH^g5SWGHG4IZA&5k#=pISwd-wmU51NzWT#e}Zso0{ud zq8zg;Me629Hqg+e=6eW!5Iv;oK+1+^R z`o!L~r5w765?tNtt;NIR~dq+gqB3&`n3s&j)E z_7;p%zW!ClM>DKSaUhL9anwarXEB;%?u%S7^SS$^JA3=NUGSzN67H4$U=YArz_8%? z6Whb+?_Zlq<49zL{Y1W4f0}(+NXTeWvUx4>W!yuuW7CP3Iz`20)3Qm&w~m~S@?Vu6SOcUA;L)ubiWeLl^MJJBY_t7EOsz5T9#~HAcpV^OKmL@jY{*&U@o$7bl+J#ZpGsW58WrYQ z6A$=14*6YMiZY9~+j6o)*%IIDbo8GC96|sWC{YM!q4X6F=lKLV>n*|~7ueqM*5Dt| zX9T6fdeWMVd)tJ$_|L+B!>oV+z)wl*pNpRIe{TH${*)*mnLnN+ANREP&)59lO#c60 z^=E3r|0i}ME=kr6O)sa}^$(Hz*N^xKk)({@b(M|SC%)+QHcSEnD}zcK`V_kU?{XQa z#s2Cvt%ws7*uuME?KEf(W=I3c$M-H{YvJ zavN2?NGe~jyC3%l7uG$LdK1Qfk!(D$Y2am>)~tVHeOiE3v-4kE=zkaAeh&`#5b9O1 zoP6pk>nGE`<=XJk+q=DxnMzK5LQP#KWA@6JQo4{*;f`s~3I9%3u2Bz=e2^}e=z9FQ zr5idlhh$~Y|6lU|Ghga+$+bP9LBot-1LAyf@7N^1zPFYk$#N(zBdz+hpw2cDUoJ{9Y0G|Bo#Sh(vrvj^cU}sS5Ync=(e;a^;P* z=?CBNQNeK8M)yKl9l3T0W3aQKOs9fc4IY($M{R8+B9J2Nzs&x>iG$uO@`pUtnVF)Z zT9u24NLpr5Ly{kYTZ!s|=FYy24U3&+APGmDc+T$SY88|@(C{F^z8gz*HE8658}T$SAP0YU1zjNIBwN(r?K&#C$3jH6Xb9MLJjS| zEjjg&HgHcK^3^}#u32SP#&$vv%9LE=$Z1N&uE*zYxvwR8tocad7HX1U5-lHwcUm%f zvg5b?KLm!53T_xzb()q|eRFX+!(*U$kyoX39 z%m$35B@`B8qvh$n5L{N%AS{!kHu+22DF(Oeq@q;h>4dd+H`NaWdQj`A@%0w|AIh@j zCstS((0jzms!S(u`K@l)S~8lLmrl7sEOLr-*vSR?UIb2E%)E|O#d@CG^`O(?yu9~} z7{Jsn8mX}8f+JZSlWOYXlKwh?*o`4TF8)3Qu}sLoJcud%KJFb4wHx zuW+<}`j2-1*AX_t!dBjxS@ThFmeyx2N_}p{$x(0)_k)&^}4nkm~6IN)A&IfN@wq|5NWC*rol{ng>i+LaFM7*Y;d@ zT5CGEUDuyF+`jHv!hOII5SRNQ13WQ0s(f~}@VU|6tF2C%*Csw!%rDez@B9m~*Xn!f z3gCMhX6C@$wu}G1TmL@(gm7?JUz^&J6ZpJ36(IMfs_ahbYc(o8YLqNPTX`S{ZWIW>lJ4O6ypCdxm+k29aBdjT3I4+Z33gvpw zJEm1V)G|E6qcZ$s8=AO>r_eE_4Jtx}Js!8akK_UHVbLb{y^8A2wXmgX;Rmzz0Es%6TZ5%W4?{x6 zx1#~A@BW9R@#7+w&d~U_BoLuDgL+hj+{)q$Zc<2w81!hQa0xzoXP&rC&+2q2IS(4J zfKEJBi#t!>GroKI+AwdIVqHsk`tM!<7*&}(wkb)gcP$DpM}P!%cDdz@LtB3}muoSr z$`xANlPAj|R+19o(o6G2TSom0OI7x5a$1XrXtn6J3z4B**Y-6EwebY1v4gX80n?Px zG(kg>gG04YURW)BN^7bf6Yembjc4yydv$?+V_X3*S`W{q^y%qk6SP=%Q5>2#77N8e z`#_O9Ct^PLtl7D_VO*dvOD3;FeC4RZ-P8KRw?vBlvMD_ZPn>;iO7v{ zx}ZnQ@JU?rx&iHk$V>j#?k@2i=dXLrqpAOy2AAcH@a<`{tl%lF(0p6|Y3f+|%LV;Q zlUntoH?hl;?*MLwP3#8%jYp3#r_(-tkxGYx-GO!ncy7?FdHHVcM<=s6pO4J(i$#op-Yn?~gBy}2GvlY!*{FBIf%(!Ga>a7zg|yCBb6rXKcm9+B z%XKDigQq5(;>~4T(d9x*bp}4hToI7!oN1$V|Dp2|&xqO9UTxjUaWmp7z|ERESX^Kc zPicLU5FwP%pu|qlW5n|1cOs{84}#~(-L84}qvu6V`#OvNd8$8)hZPa!EnDYR9o``u zqM_5Bxz6tE*3E$;=i}+3i!BiFql!x2s3E)Iyu%`c46KR{vO;QQ6X}N#8P^kZr>ZFQ zpN!3aYh;*>`GNS&c;ZRnE@^2mBMU&2q~fMxEQhXl*dZyx^4TDQ-mr}3Q{YxiTBRF_ zSzp%eM;lPI?cI>7l`tLW18$Anc~X(I^)#o8*~o+lb)$n2VCH!bn~_L(B#YnfVE;h* zi`&_*KWza4L?*`0Q-g{84mbpT%Ys||SZWrNK^u%)lTf_;rfm7jZZ(ldVo0st%(4P< zZIdinB)c?g#R+Mcy_l~tVDxskPtAh((CXL;MB#okzHPbpn4_o0504U(JaQWz9_DnI zO{zoT^n2u|m1i5`dp@VwYI|#+0DPLFH`|}n9O?_lH6L4UylD2nG*9QXsg9!HoA0^L z;IYU4qNc(7alTr=#Lh4t(XD>l%+e#C;l^lt$LvQAvuISlKkh-}|3&e_h{IpQQ)|`a zg*xc%^0MysU&y(gY)29~y?#9uT@3)`z1;KE?0t1EC`>^bn2 zSu!zqN!EoXe{@;p5G-S}X_I}5XGsvX7SPRY6Otj`Y*ko*XLP2u43Is}DJk;o_BQW( zjFlfhLs9hr*E^OQdOiCHT9a77?a}x2#f#Rn+sWq^-GJ2Qv@K?LnH^Ykim$o)I`slXu9dYIv9}KKil0B;hSv ztYI$ACoE1S89)E!n-bRE0|!_S%#CC3HYHmwt=seGxTP6iljjfjz@&JvC{}vlDqS_` z(TnnFQlxNCOrhoNqPN3+P^VH!?%LL^`eu$<-z9A6QJ_~LD4LNq_CEY}E8c53Ru1^g zXCUgAt)O9YH#CjNR#MUlrKjDP13i`Aguh^fL{W02SN1`2Tn`yVp(1nMhCgl_4GixK zl&Ej=$ew~`LD|Y{xwl{a!Kd;+`r6JPe!Cf(T{kNkZijmP>Zp05saRitJm17C*gJ4; zTyHLPG@_X2`GGO-AoL@VshB%%h7WR;sbGo)#@cs&6kRdRC<`6Qw;BY?^qurg@9%Q5 za1$(COFigW0J=;I&y7i=Z8PG(&!p#0pUVnao|2Yd>pmp%qWxz z)+rufRK}^etW&)=wNEI=Ur)*Zn3`!BnHw9C;BK^V-|;Gba&}6US>0PhcE9pRQDdzf zM%BEZb8>J$4$tKUs@d*5i?id4h*fDtqFzU=tXky;rQS`tte8E2yu)`$dn>+{pn0*Yt37&jz8v9f;we#vhQ%Re5uoS6Fh}Ie%E@3%vV3 zvT#dt>UshEmn>}V!(;*N|4A0^UGLxPIx98j&e&l}3lD|u-nwR^ZGus#K3TZ$DaP$(3)V#T4j6)5h|;_mM5o>Hv1yOrWlg1c+bAR$m(g1ZHNx%X@LeLrjM zf3nue$=>J8&dfY}o@YFv3)cZv3wlNcmF)HR>I}oMF~6Tgs}HmEbFodaMu77k|C5SG zRrdCqZvgYFhs=dF(ZrD>CF%CS$JQp^0+ucGk7 z!Wr*+Yt#AasKi76O+Rnc&Fia_y4oWK9wn8vig)F!P4YhmS$~Q^o|Sx5k|;E)%RF~e zM-$!|%?2n%Tip9ma%`j&mU(JbKDT^23{rhP4=_JqHToGjCR+GVDGd{46md`yQGXn+ z$!RY8f+4URMZs235Z;1c?AixP4(`y%w0$FY1ay+$In`?vW5p^@axeMlWTQHy;=9YV zQQ*z7HcOwP^h7frD>>Y>AN8J6(I(NgJZU149Y;1bxvWGsCpVbWZU-wvG@vyxzr}T= zH{xE`7)`YuaY~~PG_PyFG4L8LmP;}p_|ddA(WE(gO)f;o&LCqsnaPwg^wA-QkB4WB z)QKex;?yQ3dK!YWCr+SY0Ra0QM-asXpXuz8Js8JLTl~|(C zn5pzD`Hf?reE+IOPseOt%|;lgqf7cGi>O?33H71;4a+m|la}j>Ps>*aKm8j+3we$C zmjZtd0{yw|EBgY+L~W5OpDFpU82Sp$2IYJxTgFWbl2oG^5?wV^^3|j z1Q-BFD1g`n!4s2`n7Na`BM9>1hCUu8Po;BROE!UwU7m2%hmLYAIP1p^ zc9DNRCZCw#w`IvONO0_5jWcta4CPA)O17eGJdoqGKiP7lHZvu@53i(u&A44P6=zm# zPuWpw#YZK^KaM41#j17Y8-Acu1FYRjl3MFtOV}J*gOgs!C_s;melvX3x{;3knR$ex zYbc{vv=!H)Pp^N$L~A|-egc?dwTTa_NEhPz()ZpeE=>j@0^0XZ0CLLV++Sw$*~^lD z3y~74upFBcOMF@V&bGDeappKg3TjG$n%4&>lC1i*p95i%H@_k5Ql2NPtRUfsc9u5J zsECt}vmFc}|H9h*?yvjKZ{Ot+b^ZUrq)82AblT|Qukz9bhRjSe&h3}&u5|2=fnNcU z>nTiXwzGb{kFFow7Fl^y-)z5CBk!31*)hKHb8TN$O{aXxObOL>)^#*lf$^KI6F5MP z$My6Gt=Mv5mgK&SER_HAz4AtE{p94SSlI`4an5TQ3-hg<{5PNf?Bf`?BpTVsMRARJ zV&066O{8Fx@{RmRsj%o-Spvq}Zl+do{Z1B4_`cp3xrAcX;rE~U@PA=on6FS@UVgW_PBld*Wadm`C}{^W#T20NIH$kZgwK$ zpQBd^KTuI#$_uKpS~@#8;E`?xqYuAhxOhk8F3_d(JOli}cOw<%#5K}Re7e%f{#u3CJYbgRrd zZTEi?X8!^h{{6{V%~d{EQ1?!R*;|J1hF@zBizY#yEc9uk&iA*jZYC%?gx+@dKhpO9 z{+e?Ewe#`oub*Sq-iY??KN24jQ~ZklCl*2-+O0#SE{QBhS^7Ue_}@1`97PfTi!z>3 zejV~p29Pl|0+g)%=`Qd;(dqxrm%|*_nTE%_bQ$g79~pmGL@<7NkHpOOKi1j(Qyf7l z`k7%%`RkwaAVhGbP%-7wHve%$Nz@ST+pu;9)AYoRE(Q-8n!w$ETtoB`$tqTshmS|z z&+j4PM-uGlpO?HuLKM;_CS`do(+V(u`CU|Gu)M5G$KLh#&pXeT|5zg8(V&OCNoZ{) z+25Oro>jDpl8q`ZDV6i|oXlqo$TaKxoK^PT^XkeYG0p29>mCWEXZ}C;l<^d@u;6GG-eoE$#b+-PLBWNI;eJ&)PHLe+aFI~obU+GiqtC1XaoOw9; zjGJRzwg%0VX}fdtdP6B85a8hQ--Ha51mZfgkNQt!d8%GO|)B;JA0ns9^JMx`nc;c&jBQ(1Z;kbO}a?crW)HxbVw7yYq+xuHeM(4nQRANR(IU2`Y#nmrQ;PUhy zqOB5TO=3)p`$*q)9M*WTtuf%DFi_I0Xz6sZz_P|`d2z&9VH^g#-DfuB3(Wc2!=3D- z97RC&Ia4|{b(9X1P}WVMVel7l-+Wv4n@8gzv`IyMc4neN-N1m>ZD+h*3g`&+9RY!0 zACw(5Cnd;AF{zn_6t0gT>;m%r2&B^DUm|0FyT#>ZVLxAFEvgMiD7PsgV+rD7t?D%j z)6({3a*Q}+%8s`d_u5qrSzCXA1Pac-VdDq*kbI z)+G|p4H^7W`-0HOw_M-vi0AyTN)SEss;^$EzHiiu|7`4e7$wvRvo)#=`QFH81f5n_ z=3^YOopcTvow1S<>vaJmD>=2#=%C5GFkP^DEQJn@&Tmru{^DGACYIWRR>79DNLaEl zp{$ZpBik~TfT4j!G@ij&nQ{t3xoUJ4BAR)A`1i!rGZ?P5PS4^QpIq8Q3<~Oh%qa}) z`=Q}zr5_2%%Mo4MI^`CuT4H1+>{Xc_A^Hw?G1J)mzCkc~hWVcMjs8XTc>=1(3By;S z4{~N?Iq6vW=?jPJd}GgTRm%X&qw0Xtx);paz)H z(*ftmj0>f(d;jH*iD>h@?FW+nif=&_QGjRKd5V&}o6cPdpX-+0Z-bmESsx#8D0uB< z%E{x{Waa{;(1e5N1o^PBv0b6{C0zE49CtM*PpXz6mL|^Y$UfuL93h$#ULnO!Rk@i# zrL3)KBQff(ivpZsgvQ6#Y~Hg%%hT_%Erq=2%EVtZ2ywftmFK^GGnM%-=K^lvNxr|; zx1$*HG-p(4eR6VX7wt+7*S!0#>I3`e&8qi(K{WclSKsPC>$q01u2;EFWp4x-U|z^ zi~H?wr)`ry-{NqcBn~9jP>v_!(IN!i-Gd{7WJfj=6^KThRv%vK)q*-s6UjvV`w{8~ zu`vJMMUVeufrU^<29s#xk^yu7SR>CjyO>#Ih}K$Rj%4!KS!^Q|z&?qq{^m(Mm{A~P zv>E1#pY!*?wMW#=Qv6+WhBk7!4Z2^QejQ^Ef;a1zPfNI2f;2fa{vE=Hq2c@%(fby|RoDRGO^?Hsp z$6`*Y@n3J?G8~HJW2_RG5VoU)4?B(N#=fZd^@}5ZZbJ&wyncCP@yI(p*XLTGk&>Au z90xQn982=AvZ=>|h4_)KKp5a$hVq&GXxFtO9VJq=Wr;B2Xgm%eyUJ@!{W%ou zaXZ|TR>j-dvFk_^HY;&(ABDzC(MrgOx7^~Ec6HNn0x$DlR}p1X5xx2&{4h$In2=$T zU_|*0kKg0{DWTJ8TEg_MUEE;ortVJ7hd!?QI2JF$XcIh`?hAK0c6%*cOY>z2F$7 z7O2%0){7=GnAr|pY4ynRZTR6}>aF|2tx7Fz!)^4@$S2b|2k2s> zKdIEnz-Wrcqvr1TDykFy&$VakA=k#EPn&^664boqSG7C!qoOE=^MO{;%Q{k0C~r%eYV22uFb6)|ENq&HN>+Kp&IdTi3k zpym3-E6vG3I(EY;8< zw+so|#Ws_#_N~YrXYLEsFL~N0A4%wj_5qkrrJ`A-V@APrfD4rFB9{kWmzP;YFWZOT z<3&ZV{*Hi4eyJ$JUynebbbxdlz~k5L zKG=FK-!KK*J)$ky;!O#}J7v^Lj)vEh6!*=wdcK5=V+QznUOlRQ9x-2wXoT_I^L#v4 zDXEpE{@d{FL#YHM*>w&|pmhTlJ&Ww?X7YmAmTUMk%fj{?4XgN5W6AQn#fV+snt`u79cK_?-+ewc>v&56X0!a( z%#>lHiu)4pf5&&^^kN{7I=xXH?V23Jo7Y2GLd2te_t-Fl#oZO%1~zgBL)B=-jOOFk zXQJw^vuh5Adn`RnkL;R|-XB+8H*1tn+beGNIcl9PwG*B#FbPO@NG$#DjBL#=LpXm_vKtUWox@k zJcSvp;jsagXlof``EI)DfWUngHwFK{A!tyZvmCXPWiy~K2cFe*Qa?%deFgjp8Y%*$Ri_mNJ%A$TQ)5?7OV29BlNwo6?Z z1-i{FPHuY?V8DKXxc9PYn@+n!T!+d_YRU#WU2B(@xRrWHo>1L+Xb`>v9{Esc=Rh=O zK-Yr+a574nnK7MGda;+=5}mP&FvosJ3M^2EiSFogTU7rduBUy5pxTo6#VKYEZ$PUP zS>2##Q(w(nqTcxmtvd;E#bjp9>eGlIEOh-BoZNAwVm+`S<26{1o_^679u5wwTA}ta zTQF&SjzVp|OM?(Tcg+khA$_%e-KdwEj!hdbn|5ksRj zCRje%Crs885PVNWawv=ozx_xlO1oQ^*WK=O(f8I42bS;syys^&7}-Lvl_kA>96*`1 z(lN4u=Z(n6zFSUaA7N0LZm?9~a9w2e$#^eKGh;3J&rEJ}9zOE*rOy4G1PI6R!~*}) z-vURD-;1Mr%gpoHk)I7eLymZ4$5PoZ3?6^>lqu#|YF3w4>jzl>pg8|n&!#tqulg~K z&A_&O$5*UChNa#!Wf0JwRTaQ{YM>u})gJoKuTV_*5&gwsm#7!qTb9Y-vAbLxj!F)@Fx_`v-Z{JAcxV96r=# z=2y6j%oEuoz=Zbv)Er6g9^J3~wYn|_U<@_oB57>>>HIgy*F8fLnq7CRI zo!2e#LBm=Z=eTPLHMP|oOPwvM z7A#c}dVPuQ)+WQ;lDJt57Mq4P1m%mbSMfA;2#OZP`(LJ;3BKvxmv&jb(DP@B@1a2R z`6{yKdah0=Bk`Ec7REnz9OCkJmY(z7a~hDaOja#xn115Jpv&sgdznhxlyZYPC2{00 zpZ|M&%zRO=CijEjDvHZA8Pn_Fjo)yD+UgIkMWQV^jCwn%rIgEX`*Z7<-S1@4e$)k( z^^6N_atVbjEMKB9`kRU=46Q^V5&WP8W|@|Zi%xxWtNTUGhl7xdgjacHdP%n;kk4)C z#o{|Rmq4+X$AQ_VHKD~v&Zgu#o|-J!8)R2M7A`^+?vj|Lew#y#nNo(YYy|s-2GCu< z_{an7T{3n>o+o8ha11cJC$RRLZ!?7JV&11^)YJ!pG}qj-o9uSGt1BH;JN|byU-G%| zzSn;hwyeAtz@!B8t!HRfEe!wc{2#4{f1Y{nn!OetRZzduSUBt|m(%WmCr zNxEirYoyzPxl;Wr?9^6$9oWY&3BCIXP_{+vFaO{9A*T2S{iQ~+wp|J)ra20ex~1+P za+$?K)qh@w{IXIli+4HF({p7(Rs6?S^@$GdqUmNbLC#}lrfo6C*M}r-yvg21Jw3Rd z^GhGI1oY}*`ozm-kuY=6OycN>D;w93h0#h$;hLiVS?=dEXuvNx*z`>9 zq&6+~twS87+)AkHd1H$6+90Yh@ZE3Ok`St``AcM&H5Z;EU)rR@0kn%1DU5=+g)fh zs%eYy{&9Sac_tbZ8jkAp>;ga{1wG=HufvR_k*Cp{acOGBSnUAL3SPVw_m z_fu=_k8w@4L$0>VSiwdFrwp{$H!m#I(2y*?HMnZW%aGZf1DD2Xy&55j zYP@H2CBpa=a&1ZXseg8ya6}=8oqwL?F2d}60V>O#V_FI5{Z?sQB zmTV9sa^U8{d=;V;EmG3*`_YPR*K9jMGfP)ahQ;B-&CGUGz5Zq}lY0A}-TZGJwm)a| z)>P6j-|N^QBEEvw*JM%kR|Cq6Q2xmj9O?B$ivs1Gp>`tbJzHyK0&we z4}Pb&Rx2Om^$PuG$}ovo-@syx+IXwrA>Ru>=fk53iIyRKq<5qsrld;??6aW=0E1JM zldTg0=sKS+?rGx)#*(`2aJn>e)X9d?ah1{m&9-{&|4R?}IUtS2VCQ(ST`jk1Kn8_H z@7~^}F0jI(y}($bYi2OS$ZM%ywMjXr(kV;Dh@l106h3d4hIaq;zjof=ztr6qokb%O zjYmEp(U<;S`#n$nd5ZLpwweCD;g&s*C~Qq9gSt6o zeEeXUGhAM)!dj@yjdnz<#5~I>NXaW7L|769W;1o%T675r{or$A%q&7YZf7vu_&<70U3=zWdR zn&^6e&#OLH4fMf)+XPOB1E=>+5Mbj1`L@~Yj*`hvrGO|TjOL4g!Nmo<(}V?N)XYz| z+NAZ~z4uIXa7ZbAufeoYiMUd$CRgO|^6du{jMKvww*cwy0E!S7DUmp#9qTTfQspcP z!gTd{poc;mqKWPkaTzwI8_OC^bdA$d{3a4xxP3r&c6a8tz7yDLhi`;*EJ-8|L*SO! z2ctcewi7k3hs*sV`DtVY0FuO#K0{m>bfMzdgHm|!wzemEQl(7ss#9)~!Hga36wQ>x zkjLMD5=&qDV&c}n)a^V9JyOrse$e+p?=P!ps(1jg6a0bv7-I<;#(W<-zF(*Gw8%@) zo1`MhSy|f&_Xk%R97Wwovt;qk;2bRjUk_1w*{9!54^*6V{ftf+J1VO}S|no${X1 zziB(T`BBpoW-Xn3bPep;7$``EJJKbJ(~xr5j5!Hl*)E<~%RJ=h={NUxoiH730}UT4 z{vZsW(_8hYGFOOcPbf;MDmif~i_Y6z@B56Yx&Wa(rp^OBt#oH@TH&j4H;Sey%WhaY z?l22K^<=kMfVDX{2uoWrYMf1rn;Mx}=~V*-)!!QZ#!*%}C8fI0duH^lULcC~FZjix zcjoH+=fkwU#Upu<)dp?hRhGH|=NUoaxduXQ&O?p3fzEK7sr=jpzsA=Y=(E}(l}v$3 z;Vj`goo?ig@55PGu&3d1I_IsH2P-^P>i$8m(gk95 zb=-7Lbs9i8e0DBR`zZB^Y!`l$AH&zsB(lI&z31{0Nhc7wzji2~_Zy3LV0dT1clC6q>T=~BSP=kzC0iAxUWZe*2R>VT~$1v72v zLE%8O1j3djRUQB|Iz!E7+x}Pc3w>K4QMU-Eik2U&Ms>ZgrFMZ)`Hka2N#g-^Gt(F~ z(;fDYSZiYI$XT2^xPvj|yM&_mgyK&XL`Lr|K^gd~{;;E|GA*wrk}3{n@~bfJD~rcl z(%z~7Mn++ly3_>ATpjV6&U1{cs*e~q@Tb+&thGn*povxuv)N&Ck_v{()73u5g5__v z&b`p7BER_B-h-I#PB29$Nc5(CP3ZDyfvuzeB;GIZ267s0^u$F5ZmLP^3-e!`(&(K9 z36s(Xp4&9v|8NR7LnGU4vo^ij4!kp7X+B*hb-D}qLTTtW-(;`Y=pMamQC0&8_$%KPh^}4KdK=d|1z8&X9$V6kaE2(CKnHdHWl|M&%n8~DwWl1)^nsUZ{P(|#SWe`f za7Z^!KgYwHDppJLIe^W8a#Ay7uUI4v?Jln)ukqH-kL|akumZcBxZFIq73NE!O~UOo z(PjJ1hgR`deKEw`y=}ZEvmFYZdQ>u2-fidSZ^pHZWyI2GQwT8 z#oW~04Z+tB0rvaE+>H(^Z&md8KmJ#oOZEckx@{Lf(6gTzm3Sxx?Ez{~@9)EZQrVzy z==bOZy1mF*+mNLG8|q|R`0U8-@`D8slhmX^v2kjtVDxd}a8{*G5IJMEd4^hN^p zU9~KAI+g0K*YTbN4|h*l=1)Y2jgfHdN!sE$$=AO%_{Ot&nJ+)}997P$c=Vk0MNzhm z;G+C?D~scG6Js3bqjt-DO^>cguEYVcmVhQ$9`%yLW@>%G1 zxf$TP+WJzcy$1TqhwYLI{faXJX&~ffgnDTH_fz7ChVxa9tu+%QUAV7@1Pol&r)c)w z^;N^L^L#6I_1s*!0Yjy3+a$CMRj(ssdJw65B33oBVmX7Jv4ICJLtn|cSEu;50x}b> z8UAM)flbg>I^+m?eq!!#=NKaZkN_QD5nfJLE;SsvSrM(Q`UTmi+u1nFZCdt-V41u@ zd5cIcmr@t1IT70=`*|pdz73^bDND0Do%#zO_x3h-1AV@|N26;NX%JE`4bsC~7!g5! zAb{NR$ceZ%orOn>KYWS;+M1p?O{K^Oo0XG{3>VN`u*&tDgar=nmqjQjp6Q%*W8haS zF#>&!>%{+>4^qO$J@HLVLI+Uj^A^*4F!~h5R;)6merw%07TBfvpMVfuVD7xXWPDlb z{}OaxE(y)#I71-yDhd^1@)UP?F;<`q*eHX;(OhR06V*iTuj!)RpvP00_0v8aOhk_X zd2g@rhf?%w^t8aMMCYfw-H4#BV?jyzWB$HnxN^3T>v+TpY?HJqA9{LM<|`= z_qW?iMC-GrF1IQs`B%NCO{*AAvHn9XMo*1niuWRrKJOC}eo6TF7>fcsK+ za<|Kb)iMtKuE2(%(s-4IhfLAF#BgUytG|)yG}Ae9m>TAF<8^VEQl(8$cJjlg93DGh z%MzQr#d=y;ANW2zPM)wdHotfsSfz_d(DfXk(8)`hrlF1|auP%=VJ+fKjqjUfIPCZ)?OS0E7aTn-r9a_$E&kZ^*FpCo%t+JvF5FV+S z;EsKGR`io*YjBCRqs91Ez!V-M!=w>|it;be_m?peZ~TB;_8M*=WtHh;}0yXRW4gjEVS-*y*8CileTiM31 zk7gP4gW%7YU9AttQyqGIMD%Lbwu^&EWx0ZLDKQ$#0(3DxL5j_r*KH2cLEqhSOfj}~ ztg??CwHY9XJ~$dLB5^43a4++K8L12uK`bjiyU*d^(tMG-g|gmv&-Y41s(9*nrd=0d z&sbjBUdnrrrkbi2-F=CJ&kZl2J~vJhH^)!|rtSge>XMeDIH5kRPIq9NFsHO~?H~51 zi;~lJ9jea25fnE5Lj4?j&7?*cII?Mt1n>SDM(I5Eu&?5X9)_BO>-e}ov7YM}WLBL( zo{fLy{kH&MfIBz}_#2N&wt3XHU}$&aQ%`Xe-op@HQFE`EZi&R1`f7aEpv@hdfh;JS zNzAO*tGaXf9XAW2jA`h3p(hxb(e)brlURmwj%VK-~ob_+EZuE52uzR`EQ!UyvGgap)W0tns#KI%H89w?94g0Q&yo$6*s0d54JoLL+mF5z9}@ ztncu#FloL_B$bG#fEv8)ZM|8BIfbK9HjBBp-pkv~>b9NW!|<=H$ce9Z0=os}Iw;{6 zX4R+01$JghgxXE}ZiAZ$o&bKQ+tK`!WYoZ(UYf!CN&Zc&-DtX7gbR-JqwwyBIIp(s z-UrBKG{05J6QyO*27dH^UhBm7&2s(DWA!&?%Vk^}VUK=(+om3LYZ5Sy z zV;V*lly}4<`~x}AijOfc$742j>SvPE5OoeMy<9Gj+R>0Bv8c-z-h{elEvDBku~oa? zr#Lu(wYBG9JzBv(c1IFJk0=)tz0P&78)x+etWCIEwB_Gti*+5vofde!ZRmCzM{?dT zNsGi0%ih}+@;utt4DRif(CRM)$1LH&`rjku7>oS0^HvKQm?S0?PfZ3hq+E)m<9V5T zwEC>_lwQNfeN`LeDpdPc0#xlg?t99T?dp@J#qjp`-l_NT;B2eqpr`r3X$hl|c<7Frp4)4REZ zcBmK!&+zG>7GscZGA3K=A}a^hvzXX*GZPu#{XG?-ykX$oLho2j`v%6B;fmu?>fXJ6 zCT@M=e8;(vtevH_HD`K8nea(VolFNXU&+SG=x^{Jc^hAfq9>oT&#T@rstrxC5s=+hVAuo%Z7gArSksv8jFv z`#!~1=zg)}zovNxn8JD{f(Sgp;hw3evq zBykdX+vTZ?Z@=0r|CgoI7cP6FZcUEs1pvx7UMu$(Q|1q$yB?g@+!QD`x6-`e%gNDKfi<}{sbYi+PlsTk zUdi}J&OePj{@TIKM3Ci5NFO1Zp5>Z1#ivhcItYY!nZ0FeqjvE?^m0q1_jm7|rN^=P zri((lIXH(KB;3dFY44(G&1cBKweC$}unA}DQ)FJamQfJ2D(}x#XfzwAMTOUG8Pn3j z^R7wCVfB2Rw{Ns!K2q~vM~hTHVAt-vGNoV)7BNyds{T_{00 zwC~l+JcE1PEa$;iMkG=>s?#E>(|AzxBh_iAm7@jvd$HT6E`Wm4=ZsG_^*v7OKi+`z z02IE%o>Ke9IhqsU)o)V0vRY^-Fp@+w4^q?5c-zKeE;Up{wik#U*fd_n9h@N1L~pwU zoEd-XH+b`MtBTJE5sJvfQvd@u(DMQ)E_#r?vi8lx)v&hv*}Ti~{-=5~#jYli}$bFGm4>%x?{+)`)0DP9%Q|8yGhv+9Q7{tN}{<_4g8%Ge+8b9;j!ic(%w*x zXmp-G__n?)_K=}N{E(@j>tOU8u0B1iy%|O-A>t)t9-z3`+}ru;Q?tkGBjYNOaqXM% zcV&`*>(vFTOy~O(%XgpC2UGCuy8YWmY95;G`WCu8ZEhbgGE6LcXeg7k)8D&wwVgao z0l#5t{D8F%k~~zlh!B)2W^DA{Mfa_7HVsMzlIt(vE(7Adg|@tGkE&D9XYWMrZv%q$ zAmc3;0W7sXqrQ6iuTLhfgSUs@i=XarAOEIIN4lF7Dzax)E{LzHvI7G=OaIha+V-NE z`M*U%$pG*n@oo~(rSPi1akS8MaT3EP#KgeBe|Ct5nlJ3I+`QD(IEYh6YI5LpRQq}t z<7cg+(W=^=>*o=E_3h|%^SP5w__}FV!p}@-p2aMB{Nv?WF{rma6sgIzK()vGFg(6@ zF2k)aW}jO>13}0f3HTm;*at>v6*9EF@jP*;&a&*#HIaW)30tV3TtqLuF@g7?XVn_V zu1tHfZ;A>zEdQzX-v4oW{p-g6aOOM3jwfIMwnStIY?RIE1qHdJ=?p%-oxGn(gDFFzQz-r)K~?moe$9zARo;@8g~(y(Tu^{f3XiHy|yEykmpjDzB$e z7GTf?Ir~2M^5-?~tSF1OQn`LLDmBXMh=Qt1;ML(ObOkT#mir_LhasHEkaWMAM)9~( z+YUpg@>kq{!;|vCgphZ-uWk39q^Eo?bLtq>?GbvP>hZ>fZ(+4XFJYrOFqz*SWVrbuNP3*vWM-a zo%<>$<_sl8U_VN&_$#!?G5c67yIkB4OY@i|Z1*LFTXYhZ&Mz~0#G6q!JbC;E-SrNG z3Kkv`WS-5zDNEJAI2oJeoHJ;t%+*H;D;_l41e{MER=^G`=NEZ^Ny~TQ(|z+!v`7_l z0*qLse5V2+rOt6($AzgEFPPwRU&Oykrx6s+EL9db*f>;6H*?@kJuU@i@Tlkc3-VMN z)_1eo=uIT5uYnml9!KpgUwhA0sWp4&cr2^pXY#dD-85M}QLKK7Y<_3r(VfB$BB-k- zhot`m4_Dy?aS#;}J>R+RnZst-5Iu&y!c3J2Nm<>;_$+4bIZ%Eo=eec$qS3m}SaFxTi zSNc7{Lh|Pq?Cs4B$29;AWpZst@UmTgpN&5l*xG1lv$&5$6s_JM!EMiFd{CCRD9s|%Y~eI_8xQZ%XfB@8pQS+edOl!t5`z_@Vhezw z1I{XRmHLe{`ZR4mP|K{|$MxnqlVGNhaJ-Yizl2}G0ZRAQ61RVgaUJFI8&9n%HPJQ2C@Y+wFMnY(6F z=tAP1TW>uNY^C)(gn2Xefkes$S|1UZr@=t*`ThsCk#k3Yn>QC>9fujVw26Lgx*U<` zh5fpIGeAwcoO=`7!nE(lfT1X%0Q1y4*4%gG6QLr@w>g2MQM^&E#n0c*SR~|1v5-Vp z9*eSDxXUh{P>k$`0%B0=JYc*gd);P-6prtFulJ5YdAR}iJ|xLTT^?LPTOF02bi@JA))YE5)IqABxTnu=^oftybDGozKBY+5DW%m9OK-p zcEd2Gcj?x7#8QqdmUlL4XPxVUq>#oV}qCGC7?YR=X z6wk5-Pj-4JY`=be^%E@}YDs;GtDzi|3o6|4I&o-#;6qs>$;aLs%VgDty}desIYqMP zcu4a<-EKNy&sk@gsDwwgT+9%HQDQm9O zxl*_3sxCzR9cVpPy=W&767HYOkOhD^R(!3k6U*VRcTk6=k($d3tlf{6W(~?zxtxg( zEZH=hZ$2QouzC%I(wQM~vkKkT*-)Tzq`PL9zax|G&9-%HC}RVdJA*QxZ6fd- zZTe|1>-Q+4k)Of3OG-+26%K8QYM_dmJ7A4QZ|sR!P21r4PKMuhnZf zJc)|7IT*Be?0!Z1$+^V0R)_KL#LjC2donPWRPKbpvez-;DiAP12SnXjUwfS4lfGST zTv$ZK>d;qPvHpQ3PkQ{F4}LS@sJ0%W&r#D|^)Z%&7sG6vBvkLOvmX96cGlJXHY;D2 z>7&~1^|v()xIIfe=fVt+!p_4zC3p<#I#%VmHs>xad8-QogqqMo)y*zhzFYDa=YgOOf8>UurBbfk6u0CsR)45R0B~^P)wG3TUKijREjrz zkGEa^0dt7H61^3DGWXZdVW`-jj+^d8#39XmfOs;CEbDBNC(unNZ1jT4Ol(=7nz)Q{ z+KfRN!z!1nf7bLY)B3n=H`8y;p=r(Js(RP1s#Mu;mT%s8V(aFp(_*72q^?_n=WjZ# zLC@gmb(BuPT>=@edph%lu#oql@ZD`gC(|gN%q7mV16!VUN3AUvSf)#eKHJjlsBNuf zIgZ=@XqC#UtDm5$M(-JZRbFOLZc%Sl)Dq-cLv9umWO@uVXx(ayK9la98lF{Qh=Gzdg>b!ODr6a!f*8HnwXG>9E=ou`BAY5!&V$$X~xbfGNos%n0} zrja)blu5PPClwT1PfW9dHlc%5@-Bk{-DRS&&&69bM?J8$(+gk%b}jRAD;jEX{^jEgUBmHT%KFSx{+JkWu2VH;M> z%6G;GwzHw8L1JFzVo5)w1o$3?SRX@5os$bU($JZ$plI2hUeAu1!|D#UP(W?`%3(bVn+>0H9o`FeFG9ZF$1-Cn zBU$<>n_etVPjqK!o$)_NbB8Pzwc%_9jI8*g_bzecmVrsk+I640sU|MxUV7ZLgF0|wOVyyUA#VDD%#m$>?2X!YQgH*LCZ)US} ztOBr~k?yN5a(l3EGuuZl57xfVv%hi1ZvI(Evstb%@rmlH&+eBNjr!uj+I?1UX9byU z8b=ob(kwh8_1M1!VFia5J;NaLVC)_dYZ{C2dsLTL@^p#U#v zO^x|VNLl6-#;M5UJ>2^efmAY+54(d4zbeq>zA$Lt9bw#LVP)?I<{P|cx3*fPRI;K0 z;(RCWS?jQI6GRIWaSKt}8nw_eXb^<}B6XJ7!)AhxSj{d4BLudEnKVxFl`$a4t(fM7 za7j_&AR_|$<*6t#1RR|G`& zs#|Cf{nO^B-)GJnSdl^@I5_^)Cw9%w*l&pMdo4Jl8{2H3pKe(j$*wj3*|B6F8+Qs~IvYXKL0gB)7 z%Ql78y7um@Hv(nyq6K$wPW%YF?qtJzlav&S6B{>9Pn(%svz&S5`F_}E-l!`U>|MH{ z!Cvlz+FrijkCeDNWdGjeeG=CM=VX++kOdFKXU1avhgj{HcWYT+r-rBg$Bs8{vtOQ? zyPC$^8SB>jxs6V=`s=jc1@BNzgdOmu`Zd`OngN3LqB5mW#`tZ2Sp{apTQT0uw*M-< z^mnH0>HMr35tXEaulf%bK(mF%w(C_sd}*N$nSpd$4XeKS^5f+97d#$)b9DLQYSQBG zVrW0!MJC=F2r^Eq@bHMO(cPowJ&;|wtZ4RQD`Pu>euwcPsfJOVLDO`g?nG%yNek1< zSUGW!HM}@ZZYP%~qZvU&x+hF85Dl(PusiuqYm*rO1&@x_cYbnxq(y$4H=GkYTOjR@ zvj~yGa-FQ8j2X6?u$s&h6Q|!fzIItvJejF7XIf8nJR-c#A7CgtCH?Y9IBc%bVc&e? z`<1Y(5dq1vrVbM_e0W6exOv+VF^SGlP*_yyndN}98MUN*SEy(i75~`& zEOs%x`#JtBF}sNiv=NQ#Omii^61TOwpJrpT^S0!S4* zy!vYgBA~yJdEbsfuvV^uxZtb>DKl~hEFi*kDBbi&1*bpm;N6!R5`5;?B=11yN#1Pz zoxL+rpU~%sMEM@F**hZtuqmii+nJiuU3SwW^n;whT&ofpC)8~75=5)cUOnX2A$>;b zoO{Q;+vtyX4$HpQMPu&@<2K<@!Y6S8?j;NkeWxx#1CNM1p-$Y*8TNxGl>r5u*6(6mVMm`uJPhO=q;H$^&*I zq04hTUet;-H&@Ma7)1A7UK#;n>2IxaK|Q`H+Y(JKf7z%S5=FaKl(qQ}BoXWi17Gma zW+n_W7P?hlp+mC@>QH-Yu5}Xx&*QA$_i_)}Qs3Z)Kc>yqT7y<6p$K=WX*iifBwb~O zj>9M((!LYZuwoaCVw~xo<^o&?2*Mw|V7x(WSWlr$dy=?G&v59hT6vjh)3WCei^S@r zqwGHd&v3|e{W9w`C-u=pC0uhew%N0FG4RLRh5+!0_O;coy-HU#36Jp#lLdWeg)rChYxN07!89H_Izj^ra=ue+c6830jg1`{U7d)(& z*`n%cZdHeI`h2$Vyam>U=DeZ=fXSqBH zXFSv1AG(g?oZd%Q(KC(=6tSq#<`Ds6B~(;x`q>}B@^=5@=_%vch5Wc}?W z$Ecj8Q#+SB+YPWB+o|L2>j&>1R0g-XE1#bY-6Wkfu5}}a@OF#3OP20J|HvoRMicfZ|9g$IdB@YRsHNS+Q5Yt~A-g$Rjnnff-y46UTH_Upc zD!5C|8{4k@{Y!VUA-N?Cz*wTd(1B3&Ov(-_^J!cLyC@=+WPH#A^(*j0J%1FELT!vM zV&=GQ)}E6a6iyHLD0*!T(bi&-;;PHbF(Z^bATj>z>)o(FYp+M8(>?~uvA(hd=L}(5 zfzX=uXI!z>e3g;=Q5c^jD~INH>6?g-uGCskjA4Ij?^ zTu4j(5d`xlHk@w(T)~=H_s$E4L2=%TXP!`S$X@_9avb!!K`KMN=nuXz0U2iG=@JVU^!tn_5TyC)(h1UE63>sV)AUCPh|2`hl~zv+41Isz(B`btXk+8@m&3 zzFbXmHsb=x%rHSR_pv#~^4I~&nzV4)2t3K4Jp`Y74kP|JthnOYTK(gCCq&xJak6c= z#o?#)1^6OG!OF#wLA5pe|}nU6~L zY_+qd&~de81F$rami+bIr(BP%^|(oZw4aZ-3GQmE5bGV-DcXE6qhb_yto?iA-Oi$( z<&`$WGV3R%@h^YrC-tXaNj_2!QpgqeKjz6#x{Z7i`yVRBYQAlB0{`;N{+e0ZOaxDI z`;{~x|1yXECPWpBUgmo8wDEY%?_ZMY-!S1W#hbfNnH>9nM)m)47xRc!h3S1(9eRN176T#FKo%q z&js=87tMn2ne=Glqu=ULyNwq#ugNTi=07}Jd>1&$@jXLl=S%;p;b-A#syqx(#E4b~ z>}?W30um{fu&+mHE=qm*)+9M$&RINU7783YRJ7#@e5AHr#N_jeE}y77Du^YJ_HMD! z)?J`M%MFJl<%V~o~oU}Fm?h|83No-^zZlsBFQuLr`m2FI0y{v z&-P_X^9=M0ZkC2X)tzxbAXRDHZ?fLien=1jwr|0(Y51-^5Ziq{K z>!N}SJqSMmBh>{IZa*`yKc^%Z^8qw?7bkN)ozVy`G9pPaiPW>~n*Pe~=d|YtlQO$` zETB`&*~w=@8ye4H{9MvVy{eNNfb>e%WJblyo;L6;Sg+|H zcEh=C1#&b{ImXPhcSH1Y2rEg(%m|K8Ln-Wxs~jR{U0y9T?RjG3BgwyB!TBks{V zQ~j@3M=&Ec`Ab$3nEDjCxfSHz)G)w6U!8uSS06eV>+GP$eLp`ASCy2`r=iY_N=g*S z=h{j!YC~=dnMI?dd-gD_4n!5EdFY7M1W*&=`odC!A5uMmmU*wstAcFjc%I1$x05oI z$pdC2cm0MS6ClBX*`@6BT-UTI+Wc7@E#|yf9Y0B0b%su!RUz z^pX_II2SJYB*tM$D3Cu$a}|}$&q#yUM`N_}gus?Gj#x;`7ScB*b50Fq7m-aid4hvM zROy!%!Kjk$GisD4TbDT7f)BJ4nX0Xs$8Da@gl|FCTk2S1~lRHml!>}ny@y{1{(n9oL% zB)fO^xE3%b>eehhQK8n9%=3h|eA=4!nvq2?$$qD&)EfE|F$#37-ZWDSu_`nwXf=u8 z=K9TCyp~s?H1GQyTkbN_qumtLS?)5hqW$6Vl6GCtOKm;tCdm}Vr-RXQB_#&re6_Bm zz*0wVwZeHOYOYpZ!O%64nI_AL2ufk-K42X_PW_ZLCQvK7a^;ES@8+PF@iL%DthPj% zv`V&YMu)HZew5ylqnLAdq64pON=F3Qh?dd(t(x%GR~;k0XJWo4d^A*=H!Su1qRLu| zT@Y++I}P=J_WxyA|N9MjORRj_`Ga!g@mgw6S49QmdKwlKf4XPhSDW^>reV3`WP>RZ zFds6rvCz6X3~hC+WRKiL6(NV#rV7-x`n;F|ssCFYz8nIgRQ*9YS~Qp+l@*hf`K8fd z1auVXvxRP_{$}{TKc{RXTh)k(+@|l&NHRKQW5~3YBq>ii@3)mr5z=yxV*lNbcob&a zhOdfuE7*n)i22QPC}*ktuG;ko>o7 zYpSkZZ%LlyDspmjlce2u?aiaCmpd$0|81TrCB49CyUr?&rvsy{bfx|AKP?$G7xtw#9~x$%OP07Q;N01z<%A?24}Q&lWA>zOCSBn z6HVLTTQuuX$TFywVgM)-19_Zz1_GUV&3mP$+}}&lOu$Lx4^nG^z!UxInYfmE50R1X z?YumcbS6$Fx#~W`@S(Krz+b*YJpm`vQxMmk7qNqPDxG$V#IOykBR};go2LtbgJ#22 zK%S=Ry^4(#ekQBuj}gdf*g{_#AAc1xg`G(Rh1h<_k$YTJFMY4uF8OO{{+aav{$UcD zbQ75@%LqMt#~+S@tqQl<9&`l9=B2-fg zWczwmf>L<>O&9g~{S~bORYV@%`LZ)+ZhC@ktbu8?2G8bOB=ElWjNt=}k3WNFUC5Yr zYH=71&Y(DkSO0jtHfP0}E?ZTa;$s$vaZ1!1pU-{&ZgRy7X2bVV8s>o0?A3Smu-!hl zZF0;sgYA5bjD zdty+S>YcsTj5xGAZ&4}UIgQ|kGe=gJR{hiqfWbpPvc@31n{8NJv}%1D8<22c#09)y z^<0|T_tbfH1dKsV^>bpLOXK4Uea+}7hb62ZFTZStz-E0NKKjap$nFI=?DzS|Cf|w+ zNXXBXu6VaN+;!*|jajOwK3p6^oa8ZL#eyHBq53`!t5#D&Cel5JZ)0H&B5euxjX}0B zAq{EUjJSoWbQJZYqn4Ps2~K`DFuMCWY*nPqE;()UWImB<#Fh!jjhD+|!O94RSlsRQ zdKdTC02qv;atCfyH2Wc?&8G2j!Z2Q1Ol4%0RzXB2mfZz;J1!4Bm$rwg#-MCk^wkZ9 zE2Po3D}5)Xv-_5ferzU3o?u-6ZL@I?&v&WOl)9XT81EEBZaL%|A&q z9uEZD4$`rFrXE-*k3>cgbhqJ&vmCNeDhSx*ciY`TtjDcV{H&(AJm~aD(CJ{d+ zX_c-0TqVS?ycZxx`Xqp+2tForIfmYc0R7n`IKvMn99?j<74k)sYiRY4zp$~tY zz!9nvMFW#s==p@dx9k)CjXU2l1({)DPWs`J-fLk-B6eZagK23UPwN<*R<7M<=G?abdr#kLv^9q=(g`_lp26PXaM?wXno^ZJ*BPgv~c8SgWqS@i}uS; zbH9Y|ex5DWzBFUA>C8d#kjXCs(!a^@6y+*lmaoCowbNN8B zg!`ud{@RiFY6NA{s?-6WnT6V@epG-hlJU{oN>y@QUC-RzEe;5TmKkEG1&necS@SQV1`oW5 z^+;4mnmKjx+5Pk)?9IDtC9Uj{G;_%Y5&bsMuj6%NJ~W^fNu4e6=>eaK8S2$aW%S$1 ze#&2Q4!#ibGF(1SCs%&ODSvGgOw*W?vq7)IKvWG^h1uNT0fcOe6+;8xM_$pfGts@! z8)ftJTZMX&fI($g2T2mJt{L>j6Wuqi*sVRaMzfyx>yD0}xJCOJhc*Vll?pQQZA;j# zKEIv||25=)7mlOk`O!EMz1+#d>{l*74rut2crIcpe-KdgW=lT7`=^;uH}W)bILc*H zjNyXy#BaZ|N(qr{H7_T)(e3TJERlfZLdk=m!v3{N1N|+vp9s;fhTYw7_YRcQHeEh8 z&n~7=iA>cmF=d4&SI@;va9o@pczokZ2xGD>vsZx(93RX8X1?oyf5O85QRV|}LXW(J z+>K|~i@6%%7njrvA_fp1f?))m=1u>+h~w9y*CW7VFx(0pg}+ooWX{<=YMQSEO) zMbZmA`lI%VxcFzv$S-K9{iBL}Ub75fxNnv3^ z1qWLbb1P#YAc?SqBnXwbWn|yW&!P}85GbT*WD81vClCoJ-Z=>f93m)0sR+Mp12uFt zJ5eMg;2)wGBB)vk#B3s{tHgS}Erj8|Ep+H2qWkY%L`|;~9+z#yDa^VDS==rcJ{Nm3 zK!mfd$dc4Dz=X2tL~!@_dLn~;Om?6^pjQImRlZzNS#m7L{SN1x@e74(jW8&DJrIC4r zPcu25QFzvYQj&jQxy@&O9Z@3Q`yeF}&*c9~#bc4mEE>~2o1Te`&VaH@^(bwebMq2p zpc|Fq<+W!jFa3GZM*PduT1=@a>+P&91#sv=X5!Aw%^D;p=}|@|kVhwWn`l!yb|hgC zM%ios<8F-Mc?EV)XyW(&$fiEiWES^$7mRCwz+NwqmL8myADSJQL_Jmy8j1_5D_z3x z_@BseOqf>rqk*u$l5H77^sUXULAyjI<+Q_G*5leCr>~j$1o?s3P`r$JcBRF*KE@#O z-%m_D$8>-wJ%G&)Zj5Kfs5V8oi8{eWvcb#*@XSEH^kBzgUe5fO?Zl}cAczQY`+WO- z5n=_fqy6{wSeJkt1n`&Oo&Auj2ynqdx*6?2SbS@@h?_uceBtddctCiw3EIGVw?Hui z2q7V2h^X`7j|GCGv5erV3Dt&hpaX?+Fe%}s0{fz2HQ=oMZ?Zv(!MK3F33dt<<%CbM z7NfWW-3tj%X&y00_)9XtE%--nH8a4N^a9&)v_Z-RCUvXZv9-a`1&?m$+3|WJbpUS$ zd2A(JK|*JvDmcZ{jo|hRx#bEeR2PHAqt>G*gi{M8G;Q$YH{hW5+2FN`gacO&&otPvJ7~+b304c< z2H}G80;B?FG3qr`t?P8F{F?9SdmCmal5QZsKY!F^55m?UB&t7AcNn4&BBCnt4O9rE zY@lYKcMqmw$`EM_VsF@RA6%b-0o*myHRE-LzrTN`e_oF#cubFbkU`Kcv>KW_8efcx zL?NmCj8JSuU8IVn1vxKaJ4xKIn30PE4QD`Gq?Txhh_}>7vBbAJg*{4W)J3X9>V>b) zR2{0slvvc&3Fu#qsnDo0R9Xt|ehOt{D)LbAjA%$v?X&Fj8H=QpS{2~trWdw-dk(!} z-4{el{~7bM@uvWp=6?1FSppyR&rq}TSTjpMV!7UFAnqPM}P1vs6xsP6i!1bQKIHY-1n3)LKTRRp3mPubro~cC>c2_jvhQ_sE$L7|Mf*%mWDj-+ckgTkBO>-dEo0pTj_%Rz zW~M}COlIhEeYzEYJ35(aV6a$DU=;7_T?IjS!6dxkiAWk605K|TXCFUlICUP$_ z8L5NfMIuHbhB}KG9v&G{9*U>F)vmYb{}JR2Kx#HYQx)Fm+339n zy8UNosd4?q30#@SV$~w-PIa7AMJ;hkNb{O1Y1HIiRG(RvC&9yoK4%GD|7MX z3GIBXmk^r-rH-hM`J9jJ+_oE^BroFAQRkXg4QIhCIV zvri3y*5}U;&#w)Jjp2=(tJ@8`c76_ms)Cyn*%|ji5j|@?kFAMkEviPEbw2F&jaDx5 z_Imca_yN4DkDgSL)XkbFiW-`Q>J0@}PL+DLi+qn>tgrPChv%PfwMT~+pdAn=qmd)w z@CFDb_@3UBHwvrtzjaHgS*UNQJ5+yu^?(n-w<&ihF6GK~BkjblNc@I?r|nv|o4i;( zfPI?^T7S3Fw!77I_7twW>)q}`yd^R!xiFln99%SVQv9R&Ug3S@Wji=OICCI*U}E;8 zNHaA(?PTJ6dI?XNN5ftEN%Pxn-|h0@Aia?`rg}R%fn{}<~k7?_+Kh7V{?;CM#Se-eXS$<5M z@@(!e{J3sdJpby#;bOA}*RkqGzoYZ`9`xwZ0=^!AphVE-mi%mYHvKY~JlHHxDgQIm zf{*rdeeZDv|25OGU8kKrD~Q))Zt&9MS^R}S_oX%~E(-(h2v!A710fUd2J6JNdQ0?E zdU&*2jnjwvQ~hD(>As|M?X!gwmv`+`?8W5me8J`9l6rHs!xn4%7tee2gT58ZW8TYP zZ`Sv$6+Uq?lo2&%XOpS8QhYYqOm z;Lh+6dfZFD)r$s=ku*XkOcqrNLq@Y&j-_3#sjOJ^xU4*F+|XD&UFlftXsEQ3W zLUsfKMvNKal<(S0cg&Q#IG75(qR;{fKwl`(G+78Y4vQc+r7KF{v}y5}q53-)Ir1%VIAtL*fdF_D&)b>z&5 z_iCg4b5LSSk(_~v3HiWb|6pJ7fp;V+#gGITned+l5fclKHC0BV=HDHE9dX@*f`x+v zoP48}0t%by+iw8XVXr_tZc?i=iR-CwwiNom!n^2!Th}W_M~3e$d*cw;fvvWuxW$}(V z-84;1Onz%L*VH*7d6Q97Q7yNcHVzwESy|jPv|+GbUtjBfo~TV--s7-e1o^y>{x5n+ zdyuiOJR|JB%T_NJjh(bnb`B0Td9}BDCA01A?GF3Dp{0}P!lJ@0FOOU=QcP!NXGJyB z#)DH=T3XWBZJ)0bGZR&nl-8z}T+dg2@SD`~q9GwwTOR(Zn^OtZti!{{=kwe%bCe#n zdhg<+V|z$x_o&B)8$m-$u(!8YDtP6EwDn_Q#Ju~ONFTC4u4!y&XgEDR9eo@V6El^% zyX!bp>31iG^S_q%Qy>voex%g3C*AA2B7BNIV#bP0!NQ_%f9k39em6l>XUvUjRZ^ad^3vcz@T*A57xVVrn5Vi>ko77)N zhlX5tuE!C1FMY0vWFF2o@NjS(w|n|aPb_H4SL^JBEi6jcO)Sl=gM=~P4$2#YzP2Ty zV!({M>@8K_+(4UkU}X7BSk%*L*MPkn#H@f7+NlLQRMGEoUi^O_yl7UU)3u5aNV`Ncuz2Bsnu1K zVw00On0O+#K}krd2zwoUe0-Xg)1#9^^>v!v>q|Bw?^Pr4Tvt6TSP>eAN1tEuP z^w(R0g9kD*@pD?Df#Ncpk_kiadMQt5V11ap-FiuDfiR! z&7Z0Y{I04TMr3ni3$$UqBqS2kIpU$1`iLmj3GltW)d~o1-0?^QfvPaTq_FXqh?N{x zmbM`#LcpMJZ&UhC>`c@pGGZUt zM*8AtiHN7Uzg$otvqt2WLvFBfhIQMlwty4;c)Z(EaaK(<4>_L7>BUTao3UV>3N<6P zkiwuvmi-bEJOvi={rPdC^>R2frn^6a;R~eX%Yp5N|HOcR*x*2PW*?K(k@k@-rN%Np z45ch8D>N;HoMU%%g~YO1ZH=Rf zZZH?^)q@$NTr|nkY^kO%=q=~GaJk{Oc+40Ed972f3 z&cz1J=jHWYaDj_oM`eVCoGwHs%pSnt@WVkO5>02aR0ETY`vT`1=u z3+Ui80#@w8{TdIAnvi-{DazLs!W|#|MT4X~&8naiievHY(U=@cn7f8G53@Zs??Hu2LYi zzpaf8dV<7Ln2?%UBa8YPBUv}nXF}N`AsJEpJ}yyyKEWrC=>MKQqCeQvT1Rx#(JKP( zTIZ)PCbfySV%HwcMo!8V{^bTT(S&>3Xl?Yh{dF?PKp@pZf!mUR`MLLrr1|St5q)uS z-Ojoj1m);X_uFxH@8`0Bp$0r|=hjjwF1U9Llz@!VrS8<72|RdMSOX=@dsm$d4~R0T zxMZ7NhCOUiV0o2pkPb3w^l;(*)+t`|~x`NNsWgmEE3zTZuQ zM~WWL{GI}g{QA0I_@|-oE4|(=DNom-8U_Z$&g2k_u7Z zDW%cd1wn5oVhnDUlbOh+)j7IYxj!|7hl5_Ob3BB%qU@HT#9jcaUl^As1vaQ#wLan{ zW^$M>O<0mAf$J`3ug+HZP4DUcsC-DLb)+_?sTN^L3Md`K=|Air6jinC)_A)xRwNc-~88H6(BX0|FENJfYhZ>dxsJ zZ*3!J;%KQ3i^6&XbEnYIO3^n@tk{3#OqSjYHOAH2l08u6EJ%M+08lTr07XuYh>Ss7ZK*al^v9a-uI{H7E zArKf#TB8+7Xfzp6K~U!o(vb3w&gXz-H0mn?{_hefC&%K|W_b5e9C{POoU8aKjPANh zxuTGWxUcu_?=BtpTwuiRR@Ro{Y%jGp&reQ>1(3*k+e6kSZ zy{cvY9xrOZlFwn5rTN4CNc6vK?>li`+eNFhq#-Etv~Y4tx^Lhw9R6@}9wRenAGAZ4 zBXz`uOpE(piXKP`~c!0jjf{=Nv1-3Our zuKWK_`u}PZcu~SO<}>i41`jjF{;wa<2!Nn{V})U$4n|D{6B2sZo>^$D3qAlpDE=pj zBlJtZtAkkLZY2js07>7w2kH_2gTjzqS)ddicvBY>@`(S#W(XiTOv9&)R``FH5G_!Q z6#A>dB){uecj3=K7`efym#t_itcC=%+v!9h15;GtJ1wxNTa6 zM2<2>`T9oKGFb%c-o9)l^jC7u*6Q`E{j|ox;qb^{n_WIM>XZOm$=x@X#9w87h zKeN8tDDWjAF?MZX{hS{&tv3djU~zhRfA%*azwQ1AS^*guy0nU^Y0>EZjroJ!H*zvE zuiH~y92}g}lNOkCpg}Y&tmn7dNuKMc0{}K+kX>TeeO>A66EQVibm_e3vT%2I2fZU1 z%O&nH%q!nEiR;}?POcj-EpN1swtQ8s)JTcTD0gZ!vNspx%ZketaY%Bo*Y95JeA{ou zyk`05?V^DN;cl>fT~X_3bDho|i-;(4;lD3m?9^w%Lc$)Q=XpHq31uv%B)!Lu@cxGL zbP$R26B@>5v*P690;wa2PflJwfEboRB(EDtIz!HsElm^!)h@J#11*~Xp_Y)s&BbQZ zq>JS228w6*!4AG5w7 zwXJj+8su_waYf;B(|LaE@td5lVWVN8SZ5@s_rX@Gawp_4r$v-VLM0!vVnx1#1_bny z2=?LTp9#w=D#plPF2AQW`rwFE!Zb+M&2yD`xHLS#IB&s5{KFT#Ay#S7n#7gl)haK= ztEsd-c85qjdfty#9vm21vZQ|AhpdVZ&;Q1L}e0?k_eb&axlE*>6On#H^UuXKko1O z-?|a_-mUmUho?*LiDBzB;CjTcZsg{ctY%{Q0t91ybCJLF=H38NFwv|mW$$I;)@mP0l;(f^PR6B5OHkg6)P0rMpEK~ z19ZC0rDp~@BbJN5b=nO?RaEA-rlwLH4=Xm??r&fwO^v0JPC|U%#0f56Y53?as^`mS z&;h`9X#SL4E{hwY;Pre_r>vxGbaMRs{4^^WQ*C9DosAk)T2^kOY+Yd3PK+gS+jtwse{!YtYcAr;?61NoQe=t%v($V>Oz|WPJzTM$r zNkm6PrJ=X(c?=3UgI-|r5qJX5g@p}2^G!7!Z*xyiTxw{P)KvhcKiepZ!sY!!m6()v z+dmv9QQm#9pQM9}L+}bnJz|kj*lkxJtz910+&HV2mM~dO)o1Wr>CF*{)8WrdymBh=wh>K!h!_|YhzG5G#6MsMS>MF1aQxPhFkcD973;pW}Hn!b(gN<8mM-K3? z3{uFXG%F_~RJH=o>jQzWoWIi9`RQt-kB^oWO~w99fi&UXJf*;jaN>CDS$JgRdX3xd ze%T_m?WRt>K|e|wTY{QOCZETIQS8O~E`{7qVRvLG=jrOVc{0(UDBG1#O6lcT~EK4p0btPWB32dK|-Z zigW@VCrWB2i{0RRP3yQ$kQwW9dY~Pp8(C2XO+Qm6;VT zM#ivyd)E4ThOczg5n;bRqFgot#QJjP4b)qVpo|DciYC%|eYT0AMMZ-YU|u&?qQaz# zyV};)c6x*g&V-S#Z9gvvoHVSRWbWq67N@n{HLx2*w!#sx#GM{5UI}`l9#4A;Mv2ja z=m#s?UZ58U;o+s^b7NW`urkjmxVcwN^_Iqb-Bz>S59w1iHT2eIX9H|~qWnhXG&D4b z5PMm$fkkvuC~%7JiE*+S7+7Gn1H0wqRss)%$)MxnV#M-*Pe8s%y>^XukMiC53u|V8G0B42ne7d)_YLLVn zyBxJ*`Pm9#aGK}!8hE?#;EbCKD;oaj^Ooe3aKTx2KTs}78mX+LguVXU<_LfUEnEp+ zDl1<}0$W5aUIN&CW*4UY-1oSvFR8vVgAVsU^1oo>-tD{La=ssIC9o4f3LRATw&_P;a76PlPc;M!X zxp-6+=RG5*ixahPvV^MCGm_bbZ-v1fMG}dvuhhr{S@& zE+;H<&uefBc{XU!7zrV)9@NW?D5+~K`~{4k2Nr&UZ3gcIo(5Z0eJ4~hg{)1?f;U}3 zAp#ro?ureO4I)JIA=rBlgc5s_I=H#WMS6%@J+^W+2wo&!Az0tX{D^a)@5`M_;sdVL z;RF3Ag9X(xIlR8`W0{+OH*VU$9y~q4;VSfz_LUzJe9QC$&q?_9EfJ6jv6{{!F*!Gh zBJb|Dy1P#ynsf3XLPiP3Qlsp&wiHMiJq60I1r>)}p#kf$p+71cI@Z>RUz$}nA%6LM z9@QOAD0`?Get=Be7zO|rSNEalf1VqEVv7J z=PT8D?X|n_2>Eb5J3+@B`Hu>}qM}wAGt`7GTeH@FoP`5EH3>g{xa>kvIX{Yp9E;6- z0y2-5+A*2!BI78X6EX)|3ut`>edXg69OSsRh%6frT1lRWJ>ryYc^@t_#3 zsPSGU3G?nT=Zx<868plx4{Mk66$Ru+6BsbDp-kL1@i1yCm(i;{X}P(j_o6*@N(FFB z6$RKVU5)mxV)Fbrpw=G&JJi?lS&j;Eb`WO}we10?*0k=z?q z^_13f7;X${jD+m@4c=Bp)6h$9QrY-&g#o@Dj7*O&B`6}V+k;sK{)#*d@)y`tn3q?9 zeIkzaUC~QD4f{m^dh&%`CqNkX7u6Vt^-0kK2ZqJ3x~6#JDg0l_W?oPb=W zA5=>ZaV-&k{D5|KFB63RZSnIa>2M-rZ~=? z0W3Ux`}L?UIDT5Av82AYmvmVTe&ZIihx6|9%v8xzl;ibGL7eGC50F1XD_wSbl@(GM zL{%_q!jPGP8!nU2RN?0527&k0HH8E)v~3OV1Bu$=JXLEKRem-j?%E?_5P|m%6$`-w zlc5W8V$)7yzb(QN^t;RFle}3%Ttc(utFzIlJ%9I8+&e^{1K73#?MhpEdd6x4aS*)0 zl(X4>)iMupAcWO-G14Xe9|P~tBC@`4qL-c&f9qkX0&?yeqVoVp()mv8j=%?prn|#U z`A?~OqN2jWz_jrxixDG;#GbU?-k6?fP03lUUBe}BqD5B>yeXun(Xc3<~lOK1FmmO-_+9<|hu}O+l*73&1u86do&kO9=0xhK{C0`5B zF;8-^vhj@7Hb9f)*Xy$v!p%Zs{25eJ`v(WM!n}%cvy#t_PeZoQi!M$!iv*)bvhEYTdURSmeycJfeD#Q?%UO;PAItvYSUAJW-VXh( zK@mOvVZ>cg4BivOM|84XS%feAc_F8p=(6)SgdxsF(Y5`ulKK@EE$kF@V6=}aHiQy< zi$39mx$ysp9v~1HsRvPUO=czKtNxS28NzSVYYQJ!@3K1T5Bi+hS<`M=LL*Se)J9t$ z*IA$QuV+HV)hz_;5fWFSCUDNFEd<9DAiXyoe%;IO=gS_%ujnA;#;_JO^5%aK-x#+a zIN9rgmlMn&Q60^}QK_ZHd_!1qP--+{6kL+)SSXWQiOHM53FB@E1xcnt#Y0$|vlHGP5pDBnT>fB?+2On=4xTO-0f z0Lv2eMk@gHCx9G5KmaTi%;J)NYes-5Z~lt%1Dz1)PXJSdfB>);Q}X_ySAdZr-To3y zd|(j&mFc4e%n$(qM4pZ*{=?P)C_bLQsKNioX7c5O1PQZe6s$8q`L{&C>+I(tm|fP$ zkJ#8f)B&UoF~e*9r(%GO3j|=fn+<}Av!H+L<_m_;04NT^6L^XD-$ns~S|b3&`540N zLG*`K@`I28bO2yoAZPi1t1JNW7y@Lx`My1-^IsubzP z-h0#Sdu%(>U03}}-||F&!MbVw*sh+)aW=lvkp4;Jk!R_!#LAibTkR{(jA{z|TbHV% z3QhuU$LS{=&ZP@{gIR>)_GzcB)`Rh)S)*gIZ239vX#(%W^!!?yW^Qh7`NB{rq(n(r z7!*gmrGk0mGOip2H3|x{5ooh=+;lP`fo(u!FOk|8o#=c`#LCapN}u1e$0EXa>$9^S zS5w<9FYc^r0GokHEhiTj_7_STI_*Gv@26`Dsn@wV)VYwL=825X?B7|^&pJ=((mYnu z|2${}65!bbf}rI=>rOA)G5xq*b~-P0-Kwu=eVg*|E<*_)(A(3g`AUtz$YMJbV@*%s z+7Du@k(omav7@wfrd;7Z+ir*)J*DdFf|kOUamZjM$Njwi_pBClkXH zKkY3^Ra6%j8tSEalIXQztzFLgsEobg5>v2_)haj7E{;>QD{W2*jkAtQa&qiF4TyPZ zS;@$rm)ku)j&(nxG{7QdW%+$}lkyDnEWZ5y`_l#(9=~w%h1kMDw`L&9sVMSdO-RLkIOZ;u%Hv2 z^>|#-i(XkG8k^3!i&qe*GPw`$R@WyuHf%R75xo z@61G=E)Mqh6YDN5ZdUS;C8?EbRePVWBqy7-_5#SzYTHl0ii!&WUzyZ-w=;AlVXCHP zueg15v^1{S_zV9Nr2OqQ9_Qiyaex0Hv(vStxw+qT3{tWjhLi1I4TlgFvI#d1J84#- zN;%OhxJR2VL^dyDg)}>Sn5q0-*xJ)?l{m$xRi5?A}*fnanrcmO8YRz5cvcZ^QQW=1=R&7pbx(1V(yC3 z4OA0I7dBno&L_vkC*Km+#*Vv!s-dTvNgb(5*wZD7^Zy8FvD=eL&%THgiJJ`9$#W)61nMK!I+%8oS(nXQWc= zoW0lGlKj_ad3zCds!W)sBEtIyu!MiQ_3mDpi_S$NgnC8Z?Q&V>h46_RZ1;sQh zlpSH>vr^x+PfbO9)K0m-*c3cJnMhu3v$C-*tTCNZJ-4-K)eg^XZhqSGzc*Vu+_Y{! zqnm^W2|wd0T`t){Gh4}Ju2jK$%KV<#=Q~fUY zVFwHzx8;r4*l7xM+Izw_FULD$%Z~pknS>&cdgmFnMN+zLtKKhJ_f(@SjLUHppQf%G z&ak>!D!JA95d`m_(T2S_Njz7gf^k@>)+S%L_%r!NU_1V7Y?XS64G!oSk92AfVm$Z`2@od}p&=3k8q3fk!sBLE*@L z-pcRv0jdEwcw)@%E>Ru;?jE7rbykf_x{H;-FaLI03FIm@0=C~Tg$F&8aH8^D+%}TA z)9txXK)nv&6>SE-le18PtADn4qD=t zl}<%D9Pi}8caJKBA^{%tn#opzPq}oGb!AyGkKWu^2IAFy-IMUFUz8iW*jDvwo;-@x zlf6a1-xT;Y$3w`!FR`<+WyZwleAC|DR>y6qYe2(5SDBv^I@cG(jqpkZ|IS$?xlw&x z4bUC{+`~`8zIbM?fevk+?SAcKDe0F8G0KLfCl39H$dL_`GS%|2$OwTe5TRaSPZ@mrf})hpve4KA5)qg`pz(!571@YA4I&d^XQg05bsq)=)Qb59QKr^13ES)EgqAEhGejKCi!0*HBT>+A5xE!kWftXxX@Ixc8v}qn~xbwv3wyVRJ)07)t#%m_McMExhWA<2~0D?|jjIVfq^q8=sy|)C)U{ zV)FJnExY1Zz0vZr-9qzQT?3EGyhc^?!iDfEMjG$#>CL@VOqTNJUzYi+)4Ot+k&l+h z;nm-#IHzdF-nbQ=Lu!Z|%k9@687Vb))iIRPmT`J<TRP$FW9Xe+crymfvQ|QJ(jw28Hn-{sU$UGijG%fkxt$eJtdC@5{ zJMECR4O-BjnPnQNxgGggO-n)Ix-kcGNW_Cf#A9QTt0N0q#9CT@vZNciJ>MiOEU;Oj zcw3F1Y-H*T3zQDDK6Sndcsu4@U(UAXXpC(nUi|X9{|fPk%Hl}{EA05NN_b_(rgd(U zlkd#kKr_MVZGeL0yWQv<#D}K=#^t`H@A3`Yd@AvQt|d)-{0Ok`&EmkRwwk!LoV&&R z`gTGlc%0m9s_I`l6Yd0Eo=TtE_{Q;e{ONbdFQ5LzjkIAzyW?SM+qM{q30P9a$-D6{Nvo;kLql+&$0T)qa>G~ zF_vqChO$$Ac^HwLTsQN&zX(2X&LF2#e@{%OP%&Cfvx;oVhq>PEZ};tVyO zhx-oF%zWKj<^MD^Cspp%goNa0&i9vk8DLuf_i0SVF((zfdD`QcA#|S`Y${@NIzp z2a^zy|ALx3Opek2?GsoY^!Nj-eZhqM0U%aF%LM)pi1h_rP0nf0}F~L8; zGaAU4;PMaP1E5u?|5^M}I3@nWN!e$+ynm3n00@zw0DqDq_T5YTUVf-utyo@4I6(s$ zFrrpCdpB?|%}52+s^%19wz84I<}^26WIMa`~=rjHtySI+fu56!TbrcACO>r(9!@^2Ou&)dd|>z z%1iLbprkhj6XH7w4%#-0J^^k(iogw=MNl2$r4W$?CP63w;+VPzSA~%Jf1B!`NTX9U zw`z4NtV+ugM;&R#s8-nrJC@V`?fHNQu+@Ro)VSPHF3*EsIE~Bw(eH&LZ)FuRZi0i4 z4^<_eAs4c4Emy=W;PKy$$ASbzjN-@H%+u1-!%^xYYecxM;tY2U9q(!dQeS0`*IAJ` zR<|z8&omn!7=M#A^ViVRvp=R3JqkPs4%9c)|79#*=jHXbOO6A?L~NC~p_WH7P6tXnVRX4nJm0Zg)2)OMZ_<#OI6T zO+%*59@0HwWE`~=6*X4mOh*Y{D)$G0Ym_&%-84LwJeMoRrsz&01T`*pJbY^hc)#d< zepKeYmvuhXrZtYhdwQGw!lTu$ah?B(2GGOOVEQI;d-WWVlWF}by&oqnZ8y1G3@Qi; zmLG6J-<9b-=d)*hZZ3E~A4#M2E@EU2q+U+$HI-(V@ZdcxmEVs^UfV=932}E&;u?9L zJ!P~&-s&1zG68kX?tH}We+zAHZ>+gAwSC*^xOP7{UVmEOcDZb`WJ+yV&4<0ZHRgoCD$lN$Gv$SzWXQL8+>xMe%kAu3#%gRh# z*_cMEJAh$Nt7V!2NFl*%RFa>m@+a7zv|C9sCcuNP2*Rzl2*2md>Y88<`C`q=SigRC z+&AiWa+sU^;yo*0BMSJnPv#e3oq1`z+Ksx628lQIH7!fuYZ&7|pv?XTSyiki#GkG)s&**QZF zE-r5G=hMwlN<#yK%$0f)W|CRGDRws2&f5(S6qLBpL^?N%#fr_&R~;{5(6NQzI`!70 zD4*B2>WS=qygtIKM#Xh^5h;@4FluA~-AqKRY}5%864;w3A2$JeCLx3pQL#jFMh} za4%vBez_+;R|Y(S`Oz%rijV?2+JU&32rK~b=q}@&!Q3FquowvF)V4HC^yXNl#3evc zX(_99x+)GI9Z~k^Xeq5t&qop+Ij!`(@o)&TaHyIPjGNrv6ogOPes&3{_w!EDk4(Q% z6~npp&ye{V)@!Uy!>g)(BUvmvpdjM&zRX-VH!rCwDJg$_ijxmom>e1NbiTx28#_P8 z8##!52e2TTz#Qx>Y=2p53=KPq@H2fLumZZT029hTyInfwXIBA!4Fa!6&B~;2D(Rda z69!@K+s%hteKz7n^>)M|5d9AU{2w}2R#ujlr>&P>gf`mGzr~5jINb&t%CWGHXUcGn?Dtm z6yzN2ZF{k`(sEat^)9TV?B*fGBo?-mru&OF+G=y2vgaMlMqORj>dL_IRi8HJ$FsEd zZmcQX_IF2t*HK#DwG6tD7a5=22R|$n=4G{06vB7jMe?7VTSZAr)qgi^;nMhZ=3#5) zWN|H@Pxex<*kKy^OZa_yk`EIvrUQKLt&fbfa=D3>iWQWgV4~rGc@O|=@49tp@sSc9 zTUT7;l$qHsm%<4>T2~T2GmEbuA80I0a1I-IpPX>U0LBUqmMNE$Bh#T2G*o)^$G(c* z*9}oD8|lQ~8twX{A)W8fS6jX-u3j&SFv$1jDb4fyi=|&}bJG)a<5RQ=jW(?~MZW*$ z%{Z_BwtvOU$!NnA5-w+ocIbs!8A^Ti=#g{Fs3_!u#=b8gp}NG^v%E zkC;`ut@grY({#EWur5y}v!dbQu^>pnLY_izRj=fILg49KQIny|Z4UcC z0cLc7B?XEC{Mk*dQrp(tT)V^SxW-soK9h^~#H+)0t%{F?dAFxuI{FFVi1IeLJpjNn zolZAqMj|49dJOQo9thdG`{(XHD$q zr@$`y#jS=%>aT}O=Ft|3TmX}}Flg27-)GcCnq2!0!>G*b6B=R3m2IYX77Bw$YRZ}- zN%Cm+7wKs<8pz4nIx0$OI*m$vmJeqU-pcM<-BvRL^W$qQF`~(+b!r@(%#LH`3dPiC&o;ZUzR$ za-O$zt=stCi^0R(*Qjq|`B3*@r>!`h8XC8fB4>;(M=Er!ireFClE0-t>bH3-Jn`ID znY{jz@lo{JpHIp`+p8ur_P@-8e)MNPD$4B<8shVO+)AUN;FeA{Hpx#)P|H4>{qDT4 z{dAQV)%kjE+kUlH%MqkA*I~aOTwZRSI5Z^MAEdyL>~MaZfh<|$Ck~AClTdSIg;&mv zfP7%k#o0xebQ={}8tU3U+Dn2?r}G;+7Jf`tlCq9EIA=($MHu_?A;3}NppNRvHAwZ_ z^6cyc9R?9#Yur}_aNfnXRyKlgA0t>b$&!i-2wDIf0H^5Lj;Rtr-0BTx6Mh)RDrDp3fkqIVkeSx1wKCkF~?7 znaNB)zMAZzvMhef;34*+%k4yu3WPJt!W3Gb4==%Les9mfD<2mPSHp}DdxqKYh_LXU zdx?~nwEW%fbv zxnENL4`pv1l-1V$4@)TB(jC&>-6v!^E~YZ?p1YkDYz)0o z0t5)I%RM{CFhb%(ak|J}P!E`4>elZ2#6E!Uos*Qf*G}d$LYBX=yaJq&(3`FZSJl zQN?y#1vey(OA@HC6^@v}%gLG^izB=I8~w@U$+WLYWD%sKq)mB_L4bdiJ59Gtf?Vcn z#YdsHr@-~8bbMgHD=#T&E-uEsao4o4(9;t*ZojO)1l}l_v@UX957rYjrq2MPq-qL{?dXS*yC8_-3x$ndOIG^@_tlD?Qsf9LQAK= ze`-r6tnoQ3Hqura_%bR@h%mQGNWRwEkX36pu2djHJKqIKNH9d?$}`5pR?5UX#UKy9 zUZTCSGE3U`V>`bvYr8^rQP2AcDpOIdZ|gGW7O@zGjAL3&AsR4Bv5qN zpW4w7zCqbh@Xh$B>#3DYEjF7ZXlq`iQ{z+2#?8*nPT-&*mwSi(Ltel^9}I&Lq+!3i zw|{%-+D55!rmj0~7%H$y)opxcwc(|(@uBM+7%ZoHFBWE71CENF5fIeNwiRTwLDV!M zhct9lVNp@NpWo4%2SBkIWsDG0sg|kxJa$XPg1c)J_sFFB&O-_G+Y%#u%9w~Vl<{%z z=l`f|q8B4NcjV7tKJ*3j;hGAUeu&wWOFoV71O<1yMiD}Xhn+o&!R>ADav{ILCtsAH z+;$%0&k|ZMA(9pZJT8|%^Ib&dNsofgbtY4v@jqAbOQ1-K%&qRPOe@u%md79P@!poS zxjen``I?|k8Vstcsw__e+xw7vT)wWr{%SePz`Ozl6ib|Nuob|JWxLaPxVguI+$xQc zD6C8(ANHY0B1*~}HRFdMzva~x!tL3-w#J^o?W;6O{7O84XTCtWaF-y>F7${z*M(;s z!#1_>@w%~0NyT#XWj*C*=DmvTD3kXpt!A4xD$P2d9mjquz@ywyyKsD(z@~-mb}|&U z@KE((x*YpnrA+uP0`jZeVCV^;`jg?h}x#hFm zlv}gW26!$3`9nDzSbof{)O(rJ@(NM0pV*{#Z}SwSKf^NhP~i+*5xnM&g?_K1VrVkA zxjDsQ6ha;^zqN{i0UM*0smm}i6=4}0^q1AUj zf78C3>B&{s(t7Z@A){pu7^qcs9l>I|O%l|hlE0%R?Fu36znRs0@Gvm&Pd-dJG8;D2 z%bt)w_0c?=@*Jw8JS{(%SF_dOizYToT+`Q+ zH$9$eyqoXkYTNh_<=v5xzny<9;-!xLDGk}JQP;=_tu?6pAiF_K)0$k@+4*X&3yzI^ z!udHeA9wjqz-xUFYj7?$yX%GO(riU)&5dcrmsQNl%4Ts;SlYk_=HwwA z5{W!XSy=|T3{1vWZKsob6Q1g%oT_@!^B|SO6)P*$UbiK9@l6ovA+3%|#P`qBY|3q~ z4Yx=zh75>_F0%g2=Kw?K?jB6g`OYE=^l<>7Xw6GQP#Kd%g zqH-qZEA#U5vO5EM^LoYt)HOr4AhWqDr+R{eOKBGE&eL2ivG$%To zek`~O#}+hpT1Y+{FSm8yZz!AD9u@G;|B(-u@%6Gq`*3Y|fX=Uf*UGlll6OzTJvcmJ zjbkx)^D;&4yElQ)Bk#rMOT|XJeM)|hwKWvU4!9To4U~LSHqginxnx4~jA_YvPlTgq zadj{T0~;de)}$}8edEK~*{3Ngc6a@$S%#%~j%~AMe(m~Vuih}Wuh$FD5 zhTiT6e2ihuAj#6w)d+(USzVz$gT#301tFgqBk<~ghkw)cdgjV4IgP*Gdwv3$4kY(( zZCZ$6&Bi|i|$nv(rqUxAG#AbGTk{qEd?+C-$EoOb|HdZpRXzY@EaXY~MMRHpZI zk%!0o5K}Qi#|B+rXxQD-lgH%^t&aBRM>Lf}-lu?_h$LBYD_t;bNfBPBF?k6$IcHi_ z9raB!#!i{T9YL_-=QAfuKz1EkZeG`F@IG&Ks(n?#-hdyHKK0?uOF;e(o4Nn(_73S* z*vB(GD*?r{HaC<5+>E6LG(DC0c);_%5T01JI#0>aa>~o8ufMg0bq5P$$eI17ko+kK zrP9o#DNscHp@aNtt#VwAIyUu|KF? zW`IXRaB{95_*KgYV)8#hq2!0oJhYVMIs7cTa-)7PBKzHIX4K6DH#%UHbI?{hs#<)* zULZ-RQ#T>`bK!$oEnb-(ZX8{UUEZCx9Up)EYnM4_BJmdylRNb6%p271T@;O5x? zbMbeb_qiP-#uH4Xiul~6HU;q|Z#Wh>F)uRvymZ-l zad$whofb6Ab6?ipxGUe4T-qEcZt!jPss8AGOC8jN^9~7#vs;G76rnj|Bobee-`41` z>lMk}JS7MbqEHC*Rx66Al`rBftX&+f1p2Z-ji&J#KI*nh#jBBr6H5JL^p}y)J`}gQ^Rf=7bcl;W-EcHu9d#8h&}$Mx zU@st~aQxbpfuM(iAfH0*o*@&))U78+%2-;!=Tw{Tu?fnV5;r_x)XmPKcB3awDC4Zy zUg1Juk(2L}R~e!0 zY=n()j~mF{X0q0t79GX&RX)UM?M91UpLXaVy_0{&gY<^hzy5EgM*ZSF@Or6*pY4m) zs*WHU(l8XRfX}Bl-o9105m05OJi0Zv)0_o&8P=&F<%~Xr4sx2acfJIWj z`QE*8ZT?gSfl85@fVv{(s#H(!_DcDIR8N!9SniDUwK$QzgFQayZCXU!v%Zn$ncknK zv@w3R;H@^)d+ZmDlU^J*K!S}H#Zh|_lA6!MniQ&f`>)&QrAogl#K?*Y?F{}bP{l$h zmTkP{8K-vQfQ}e#yN@k}uU6K3`qg`;pg>5^cSU#}Y>`fXzf3U$0shpy<6P!@3K3&QIpWYYNC?{ zFU*Ci&ff!9)gMZb$Ynp;-`$n35aGe%M*Z_}!^e;7;z9Of@IOP$m>N#`Nf$AV9MzYj zbq^*jA8N{hxhxm?yV;`bPXXJLl0TACK%9pRscq0A-W6)YJ41kB>$r-Y@+8Mso$7f6 zl9_ikV|TSMRLhw*KMT?=Ri%LGdCi3LM(B_Flx#KSRW)8U?#y|EJZT@rQ~c|jmt1dr zmP&7eCeF;um|YPgNdc*A#hW|wA%X_@dqhGyUC%l!f z4BST4Z5{H1dRq|Pvsmba&kXy0;=$+kk?@10Z^wT_8aE3vEAY{UHdZs6)j z>3>?ifbITqcu{Hk`VY?=(=PHJV1J2Q?BG-W@P(!lewDT^%^zj|oD~ukpvMil>}0a~ z^R-F%{@-}qA#4t+O*z-kwHJs|pq7vjeru)yGSv%tjPs}wq;7-M@ zGWpN0j-kMeS2FudrbEi|Z?jK|06J{7?T9uLMM}C%(kt`^*RZy+`C$0e? zu1_*slSnJuao6_i(| z;EmWQJksd-*xoHzOq`G|^+5}OhqynQ+|Ih%+@xH2<851M~1z^})j)~-~l zTtkG3iSF#=>U?vZWv%LwtegMC+DZlP4cLycsX8r&`R&saR%>W!b-A6JF&Q=WgcUOa zBGN1yD|hNheNxg;wMwZfoOMgH6ACQ_IR#CTr2-=kJStfDJ6ahQPS&ZZ3Bx8R*ND)_ zlizoP0Fu|ilmEh(K-P;uT$2_ks#VkyB*ZngId)uuOESPDcq%j;(X^E&mmeM-nFi=r zZLC?a-~lILCo>a2Vlom;=d-^MCCgXEXz)!rOV7UBZca_17#QH>6?iMg$B)v|@O73e zn3zbg-&h+nV*}1t6p?za=Hh$`E@C0P9wK)n3a>0ul7pLm# zIt^OIpHj+hJsPdQ$NeQ`@GQg^h72J=4QeBNWyrTqAz%>Kc~(hyafurbXBC^mL@7 zWQqtPYuZbcr%({)Yw;%O!2%)=Fwqb?$TTl6Z@LzD=de&isHCJ~^duqWcv8-M1?0S3 z_9^%A9pYvih%ypWf6M zG2@Vvn?dv7)kmAD3?Oa=5CV|k4O;HuWYa1NJZi>Rp}WSn@*oink_}`C%;8x29+mn! z8WY)-c6}Sw?e+ES*x2z_|7ZU!gDs7O-*4J6m4Bw0vzQ8bj=7z9Ps9^z8lTdeScd<`UjGL{qEIWs z^l6fFcSF7m=22G$@K&`-Th)pMEB2ChG+9Pv^Qw780&M(XFMqEo%Nof~2nY{@I%1zf zj0R7Sl9M~uP^#Hdh6fJy{W>7mMCl}(hNF!0cV|B`$}K!nafNg-F*8qWkGPK553(?O z-TJGKj1Koqahd4(C8nGH`?4{*f(scxbVnf^E}|dY9qXjaN7xAU^V4<$*|GT8_#%xX zsOZzHpo;K;)FByOaO4y&ue|%OiBM9CLy?+v6X2}{{gj73?W(HZg^0F@Qh(7yn~TU0 z*QbUJy0T9ZVr~Q-f;@(|B3-0OXbk;>_;`3lwl>%0XPn)=h$RX#5#e2%49vlX;EjwN zn7CdknwqiidW_HNc@z)FV?urp2ymZYebsv}E-ov^b#1qF(2fyw*hLS?JN=G>2d)w7 zLFoYsiph%i$=Jz>mYf@>h?1CIuA+JruyJhx-F?Ycr{hqWi2*7xsAVw|R)lF8eDjhN; z&kngWuGEJ=DJQLT$MKpN${%WqX~Kkja{0{XAR%JFm` z7!wlST2{NYS7Agqqp`GsAWjPuE%>9A{Ru|~!ZL{BJsnOKc(yH0EYLzbyzXFf85Vxa zy@rC@d7oRB1v}LRD@S>>d5f{-jqEuYuedhpU|Ws|C5J^&HX(?+{&SE44gtuDFB?Jz z8e^46@)SM_+TbTQSrvr7$`W8=u3U9+*S#K#p-@PSHORsVFaV^ zI8HfnH_3jK)w$I2xjPO06@x?Mp9v!vCKNqf*o%`FjZYM{7n@<&DH> z`2Q(QkRgFmizu@K=FgGvdi}B`wOaOu{`oi&G@yeM5;XeDXXSPaIFCJ7o4r5rsn2pVz*YeW&fr>zW~zJQj2~Nt;WI4_-JKe zVYRO3btV-X|L@5%W;BTeG&*Zzs& z;jp$K>ZamDxUoZjq&`x=+UqEY5JaegyD?RQZ7W=lOB%!SdckDvZ~SRvGX!dEG`0kJiTtdYK!G26e4Lla*{qC+6mVCn21hnDBVoE(J^(=+cRDGIG75TCq9Q{iB0}!{Y}| z!=<0M2^6XLn9r*vm`a7RjQww|ADW7KiDFW<_>w*|CfB72w@gj{6dm zFG|1f;vC6~{9U8xsOYdnMl_VxRNi$I$Oak9Dc7#7XC2yV)KHBQfwwa>?98vYQD>Cd zI^FUKCPY#=jsesyS*2q|4Js8mII95SJvZBwDFB zF1|I;-ht2Q5^E+q{w=z0i_1p;8Cn8mcoIgp5(_KJNGuaG7CdFR>J@^lM3irAQp~G5 zC?tIIlU>UG!5U3EX-tMj`wNV>NK*zry2nnK){LKuhc2DmIDOBUKfT&h)%cK>6a$EG zLW8!xU0@m-8Hwq>zbs%4uhmSXr*CmDeq>Zi;a;x%E**pb;%RDRo|se^17blf@O5Rjs9O5|DAKL!41Z?k-kb}XANHl75FI6>qVvYK;@`dIW z8yu1bJ`rrBMY+6@GRY?}XiLC zOA%IjZMqzpi9z}$UD!sh7CHtvdwuVoC>y8EtLd8{;y-g2QciQeGumwFraDQ^>gY(` zPac|jo4dYB(P@>OS{)f3#m3H>EhTJfWh*bQW=1i+pS`AJxQ;YF5YWVDnX=l_!s|`QuepKZ^_*ht4h`CX2Oiz#RIdOhj-zd*9 zbg;tX+uqsszK@}wK8jl|DK0hxW2+HZTVtp^-xDf{jjeF)) zU6-@A$71%~ZrLqf%^-7yYEpsy5=&UbI}j>?wKE^rY7~{VR5j!a34nD>3pw(EA2$pmbG6NQ9O1i(MddRJle;JNQaho-E}@5 zaPnrE@IKQCRLqfU_;5Cy{Pp-auC*gc%6cG)CZ|onTM2D?2{xp0QC3|hfyh~L&CXMD zhE+^dw7Y)rJBLkQtrH-X`?OZRm9w6dp21b3k z#8pXs@CEW9n>K!KQDYIj*chGUwT_!dy6%uk|Ct2^!QnDq5v-jlYTu5cktPo%flV8P z;pNT7SbvK>3dAUhWb{cgW|~h6mmnXSkSW+7J2rmF-2{F8UWrP_B=i}lnWlx(T9oB* zx7jEV;*p|d%%VtlUX&Xw#D-=3{k||HqYl;E(#5bkrXrI%C1AOhI`p?@j9*5XLlrAIfn(RrB%L4nW)oN3(tS0!nX&I&Dv#8eH z6ucS{Hyi6-vqhJAXBe2MQ3%R){n(3Y<*DTb^^B;M~3aFC_{v z2S~rtMASsGfQ4a`Id1Gp^Nh3UUEt>+p!)tvw(K}D#(N?t%X8wibxp)8bgk8yg8`o} zf@yTHeeXB2qz{H-SIkTN5ObW8z8P`6>+bSqP&Ty?oRxkW$P~ zA(M~GYph&$PWhkT3Bbnu6-bB*`7;=ND{W07K47v8|GDo#JzI_pGxV6!49C>@lF@P# zWph|R;SCc&mgG3X~R9B0VTVhh(`AjXeH2U zj3;mxPIK*~Hhlv=^opqC#(&z|$;s|j02y@5A=0`q=KkSA&4DYTGyh=`x3SEh!xscc z1e_u}uR@*vf<%=w07L$ifJDS^H;x}9o}gdlC>ZkI@3+|xGLIi9d<%V}u!#RuzMOuq zOfn#dCjP=KWxRpHSNR2x>7Q`N|3QUzuK+g+lwdC0e=B4#)TDc$@XdSXG7S22bgQp` zk0Sa>V*c60cm-HpWS<u?NAX_8(*L=K`uk|7N%zFe!Xaq=Ba)8w^4!0T zFm}xT0qYY?SCaD_QX9?MBhUEG%k)RqMlA@eh*P*B9Hpy`qDGLuqU@Tuh3(8wHh9h2OhE%gYH{ z?s`iOtp-K1wF7skHn|3C*ySK$Wsdi^%^FcCXr#K*2K|6v*@KartMiD%|xIsuGMYw08baKMyo|SZ;R4-9X8vrlTv(t%fHwGBVuX$L-}C zNl2Jkd46uO=cpSWBTf$t3~XsWd~duEYCJ&jIV!@!%F4mPjh)-t+|u(!o{)zty~4$e z^tqxjpv9;qlJph(KTk{&G+0_)^u91359SWl!ovK(vtJ+S!@|M>z(%31ZxB5Ip@iU% zPnH%Iz;ENlKq9@dargH@HHA;li;X6ZPAa(|{epr5UbkIbLc)`bmIoVqDJgLe4@<2@ zC_*2y*W9+psgbxdIcv+FH$HdadR~s3A7!l(A}-)ynORses|N-KqBA(>=cj|o`)O^| z>k#psot?Y1TwG?_9v6R*sZkP{fZHm@#_H(kpvQk$N8V1`O5ZXdnxaeLAm*D~T)d$a zcmbMLd?cYZ%FcNM1vNM@*v92>*t6rBPeuXAg?(?E+LoJqeF4S87M1}X@+t$NsfY-~ zc8Bi`=z%1nZpE#oC1<03HUubt$7ff3A4YT(PtIM_64l(?=AI;wtR*U)&a%ZWb{>HI zdS_R?2A8AfE{XE!u+IzS^hoVY*#Vu|Rhaa*4-YLD_Xpp%w#=!htdH6k=!_yDd_yGQr|pS; zFB3pI3J5Ym%gf3il{ zuHKY2G`udLDQ-KG-oG?&Tk(H|R2v9Hm_qIk3>0nT%OUJuIoSS78*=*lbvdV#Y@hY> z*B6wPMn*@9fPo`U>|ZZ80lBkXrI?z?b-kVIh9653^%htd8Gl9n%T%qCHv+9hCe23f z@9u=Z{1Q!Ym{WbdyVyRSt35ggA`#10POq;&MTUgT=xjs@1M!HWST-$yc;4K~f*Z(H zyoAit$bz}XzvZT+v|CfV458>PZydLTVLoyYa0^|6J|Baq+>3KB9YzZBukWfYf!y!BX$Nvs5b`V4!qzbzON{l1n`x?TZcx38n*rBkF$C6%=62LWuSPYhO)Gtq-`u!IEjh z)Ex%zm6Y5~OjJZgy|kB}ppdf4*C6vt8uPZtVsl7xr)g9(E6?_>0^{?5aL`VU&+&r^ zYfJc#y|g{Lb60pOgB55LOt0bHTc)kiICIe7P^RuSJL7OfdPM-qBi+oCK-h+NgRL)U z{{s+m7ZWekAynnyVD|<^%1{_Q8S2fO^?>N;=r-pE#ANBS2RF}?rCRi&o70L7A%8IS zyVG1!1_o))E%bOSD zGcAUU#kQnym?6GZ7Zw)&;EZtmv{+~Jb7VlYD5s?b0xh^c(C}>Kq-{w{O%Lgz-OrD! z0nlty0tUaU`^Xf^P&FVMHg&#AKAqq3ZeTj!dF9#js5cUd^p0$ZN(4>N?K>HThy>6I zCSRDd^J2Xxyw%z-S6YcAXaUpruE>S#2O1tKuP=zV!JksvfG7nE71isuDo65NC&i_P zs<^nPr>C9W7$N^{d0s8Xb-&HN6E|mfvKTE7Egg%zS#gx&zT|mya_#l?e3b*y!>g>i zx|{KwzJf;}_%%|zNmcMYKSfOpGG;U@8Ea&uZgWdB(Rs5owIa^VqqbEaHy=xTDX9N%PJScj8QI- z0P58(NvUU(lh$;QuKn(FAQTe8HTyJl$&(hNRL_N_*DE)pUDeANO zx@*X9-v$-q1Ot&2(wrGjThbT$Rh2{!$hh@<^psycyLu4^iQ5I689~LQZ9=!_7hSWu zJ+5?KFh7SReLa>CVB~hKq>-R|-^ak*23($*Wz0rB)~-9Gb1TRYR#wT73=r?g^ofUL6aV<8esXH&n=|T! zsrW|bnAdHn7ZPjsj|}%KJ_qv&zgW?4DJeAYx$oVDl!*EGTIyNWkU>x(pK^_}lCI-9 zU;y{GX#AL0fvvUm7sB`fC9HnBEYt=*duR^ckNn+86v7u-t#+c_lGb8E&JkF7oYZ$5 z91OI{@kXGi&wG2>>!<~=UsIA(0WB@0=@B0{HSC?r9%WpP687uJraRK0w{ex6CJj|) zfEpLngMrAtANxiz|pf_#rJsuQPn~#@${f4@a(|Gu;UPQ5I;NKF)&Yg}~ z?2P>EAtORuD=TDdmmP2JW+V}o6fy6s=#;si<8}yn9zFpbU}<2G@1M2IjQ@~vnR#EpQ7hn||NGI6-5)7FVn%`6mC&fm^ zo~lYoO;(`bxOR^I*bQV@>ojBG`Y`E*efKPQSApSP#a{PUli+et2RyYQ2 zl7K6s>E~FtFLh$r?>`RqM&1zc*58bI0TI-hKkbfBPZPIQtC+i9kF5Zrej|*`G2d+B zZNaRyw{mv)!mjHny#ekQ<)$c;_l$m*(cxh|1O7C#KlZNf=~CDYCK_sI9-UzDePJ_t5tfLAH{N>eJ16u3zb8UuD0B)=SSytx0 zXi@5+)ru=Kx6+U6x8+N)KOvY-U_2>5#jNTt@roM4TfAB%1#HJrhy8~dRdzf-ic}lI z{Zu|UU$s6##aP{f;2894mI`##RzGM2Llrh(4-C2;5}MY(m8CAGwoh4UFv4x@l|!M$ zQ^BI-S5;9Jjy?9VfnwJs!o5)wc={R6a47zWY-bn1N29)cb&sU0Ro~953_n;trXT}J zns2g$H4dwsG0j%vwvWK46SJqfj2C$ME0O!!n$l^2XnzA`|Aq;s$@;?dqY&@)Yx#x68t)WCq9O$J zG{Y78i7qV;YvBOd^wB_?g6f@G+mMoG=NsLRc8oqmac^}b32BOp?2~J)7Lt%L6Tid< zPeZfSI3u$rW%qQ`upbloXg9`r^_j*x7pvvm)ym>l=pTnOv<37p*4w1ng{t7ye$xL& zHBmiRd|Wn8bVp&Dx^=cO3kD+$t?a~YHbZMHBIamm*R3yvMMENLsB{dxmjHxap+dw# z0flcM!bk7yM28;-TSQRu8pf!E54oFhG-{}>ze5`I@0z|`JhLi?jgxGscDoBdzj!${ z%Lt9Y^Q(n<2N>iy2&oLmxygmcvapfltGakGxw2 z_B<=Tr^py{M?S|tMb|>VP0T$C0;cGpC+`N=6+k%+hC;ozzC{BY9b*}SNQeLnFFdo>U8JdU$c?i zl`JTXQ#E3&U}ye#OG^|MLn$_Lm4DLw^o6h`Fav07XFys8+rYPp=^~&H0k!U*D>8xU zDXVXHb&tSnwHtZj>)TUlqL47*oP-sPKvprBNZXsY3gj{yFr3C8%w{cilUa#lB#EYV&>|hc1N68*y5g3Ktw-3VQgIdT6!KB$e31+*4*{ByJlFgf-5(FXj zgv3PKg?cc21D2DDFZpA^1&M9N9_1KhpImnPQE2BMpJ^CQ7Aji4f8ToC8vI>+tG~+} zcZ+LS_Z3`YLiPpV0vRO+8WJQ-81Dl?Y}$Rriql+fs|S#NKlooVJHxO=Y-&dceK zX4~$5lpWiQ!*(F0$QY(&Y%XyHA4XJvjx~Q@*hJWT4tqZO(8^CXo#_{$sci*cS!-CU z!W|?-LC%C8&j~vo2qxqro$xszJIf^BQnf5-=CKRP_b`s`)d$Xs7t-}{B>2z<&2d=L zPM8m(QSWo<`s_()HzXL6s*jHM69{Ta#CWLjKG-IRAR`J*2(;^IZc)AAVpFU*8D1(L+ju%%^K3$vu{6Rt(^84$nqX5gK7LPF7(wx9MDuX~KNBCVbC zhAIcoru%0-BS!K7vd{w zp0m4q#IYjC4-?Jxm6Z_wa5e|l_RAR)GYKAyO0e0-#Lq<*V(<6^+Jw1aaLr#LP0$oc zZK1D9b}vJIv<5WH^4uKrh+bq#47xFr4EEE#ORl-yx=J_;&zZ?hGGV-StD1o&D|IGw z4X-SMvb6pXiH>*|`#)@2P)>hpxWU0FF_A*&TDBLuZK3W3V007}71a%+tuaPkx=@Yq zRWvl_OBd^UzI^OmnBltKq1UCQp>dVd%f{Sw7Ry(sald(flDON(7gl~T!F~}nbSbF| z6YZ@12e}%d$V?teiC`r6j&o%W$Qy-0El=G{%M@Y%oiGTXQuZ?EQNGe4U;28F|6oIY zkmSOEx{u^?F5y2M9YEd?{LLF`Z5PJUI{islc!P6{KK@3?q9_17&n-`RWkdbOQ~n9% z5A5cz2xAxLsv^n?)D-{z_90K$ z01%$Im?*@5M+N|bFu*3$PK`~hzxJR4U~oAk*I22sy^NmUMsoOQ|^-2LPbe zyw>o4DIa?Q;ic{W`Lo$;*}=))%->j~DZGEsSEogzLVAV@YyBNX(}zPyF_ynh3Iy5v z?MSQI&OfBD&N}4NCmT?swLlmE>zx4oI00CQfY-~|w|LKlYah!O0R1o-^yq1+RU-y9 zTRI~M|Lg`8Z2BvxAgrAU1XTOngPdS$W7tpLWOm4 zV&d#`pdJrdfnpyPCg#D;j@f)-jfb@K%%AJ;_XOEX^)S{^bHFi@%46#*?=bieI>zTd ztU?I|1toILMIBejB`y&e<@5V~;K>QbE-DYd$mUL-4e}4_LV!Q|ULQBzy?*`e8}^vI ziOCkgerq=vx_fyooxGg-m;3INasaqkbQ&j+bN#rULMmZtVL6stp9|z@x73s#1tJN2 zjtrasKe(8!bX}T`PIgm3cIV-cjFamP=n2%?`aK-e)5GIyqMBEYZCl63pgI84>gg2) zS_JY9i~=h#)=q^;N1&@H)lVr&gXgZ5rz6nQ!{7joXgjd#t%x90{1q|NQ~BNB-=B|< zug+$L%l9nKjwW(za#R4*2ohP^XpNVi-Q{W^9?Tz1Hd2;CE~D2?ti??YfgnT`NZq&k zY53yjS2j`xmq@?d;E5DptX8&raMI?zQ?>;0ha;E_{e|He9&RLh1B)xT?NPL2p!E>3 z>k`|sk54O*C8?k$ zqOd_D|H9+$J0H-n6DSNiAb0PZYnoh+IlZ?_zAs*2(toZ6a|W%TAmMU zIT9iw!ZSYxQ&=c&fL|Dn+$(%DUjWb!P~ridj$p!Et`?l9IoHM(wRZOQ-1jf%?>~vd ze6e^WChiDD7(6H1GaVifa=i-d)X(@nydviOz$u6t7*m8ORo#7aC zm9o+jQW0L}FU)oeRFy;JdTs6p_CC)OJ>l#ngI)Li9UVd+JAjF9fPb&M;=j>kGOB%Q z_!>wD03b)WOrYL~@y_-->P4tfB<16ydq)N10Q4T_bG@eRZ*2e&2~H$JzU%Ajgp4q0 zwK82m*ue&1;~hdmZ7&YCK<88bH*X*Xv4pO(toAVC-Lkee9s@c%vwOlZfbwW{e8Wpc z_@nUSyLaz|9PpnHDU|}pz7|qZX<%c=c#|Hf;O$?%DPa#9L3^{_Qc&N`?1OjT_i{0+ z>3Wm7n%U%fVx--_NZb?ZVNN@NwYbPb8yS4qS~*YIPOz(@@nDpYzU#R1uIUv&wAVM(Rwoa*O?0m6qX$YmZK%bf{TtIZ2`db z;UVqYi3z2wY+8BnIJ<0AE-ouHlw>3}y3*3;X8-_irPOLfmj$p+1uW(@wvyQcv3D*Q zP3B7g34VIEb#jCfp9BrSkC-OST8@<1lCKAEfd-teM@#7m$@^u~IPj(w!Se zqdv(19M=O+n|_B%fKmJ6SL$7<4ho5Pce@Oe{A3XQT17|ihsIpw#`JYU?V7wP5np+4gnQKQd^RXBIVTa&yms=fxA_xC+!kkVx%p)wpl;e#-0h zen8T^fH1OpZr8Sxl|`rxlr&8^GfMuDd(+t|40OVw&?v&N!Xh&Mqa@f>y4`bFM$n@rrJCmv<_bvILcEOAhOso;TvBUOw0w4F~9 z!(6)0PxmGIP;<` z%7V?j4n@id_E`c!tb=F#S%@^Ca3g&EVH9Mnv)rU^qGfV5wfgwqC z-Wwq+FP9er($eUP(TOC_K%+%{?Dg=`lin~W8B+WB=tWJw3?mD*!BN@K}X5Krj0?ynRXfQ`IoqJ9yeWlXU!>n=R|$cmGHy)!maeSDlnOk6}?qiJoY5VIa*@x8P6 zIl8EYrxs3(v51e_YLk$H4a;2ukwJc?pH6Wow?aKC>R00NWYn)JjmIPkhLfM(W%Ay+ z322vBZJ;CUm1>l#c?SN{+RCqr7}dtj%xpOQ+2r_`dbYeRV`X4V8olfT4LbX^@7Ei8 zgs}=8%7ECL0?yof`}K~Yk)OTMn|LIiPYZ18#Q`B9eQ-@KLjORtT;eT2KDA1C{Fdmn%u-aiV_pmfF67;P)Gnpx;^|8Xz?EYIReO1Z*_e! zkT9bdQV46}w|NHo#M}__ceYqOr*e4&5_3`_b1T_9&BuxPw6?AnE~NU|+tJZ< z18`h#%$u{b>PDKr*Pw@7dhm5h%S#7m+FF~pc;65tOFm_1a_$qZMk~+Et`HJn%q+}& zc3fT&@at#FJ|8Fo}7>zSo3`^V8D^e)WiA5YqEp1oHB!)+%GimAh31_v)I zZGp(9V`S7HIZTB@A`}`B^veC7S8usgAq<52KHL^6*Q1aKg{~3XLRs5i|5}2S3CXm- zG8Gc?uW@#!7)`C=1vIYK{ov5Co$IKB^qbAe$;nJt5jeZlqSJFz#(T1$#9kc;{7f8F z>K0zUE03--<$T=aWDEvY33}ZkCea`XQL+Em-dhLNu|)5pL4pMh?ruSYyIX?0yE_4b zZzQ<8TW|?Za1Vsw1a}MW7B&udI7!Y)z2AFP_rLe*)=TY5QG3ty^mO<1tZ%JveM=&P z&*^C30-uzWwDFrDyQ_Z)0C_sU?G)4Wprfu5W8v|b0GUL{VauCP0k{Do($e@kUC(%A z&ip^*Vh^Rh^6i#Dtjevq`Pp<16sq4jz3QJ((A_Pe9e4AGgxcO3@%Y)e`g6{SNXyp6 z4903}DC^=x@u7rud$Df5$;ol^UW*76R!!8lYD3kNYJUBCueexBaL$>JO~)40#zh;8 zwuX9hRr*X?W-d4gT3t<@le2+hKn;!rM;;}5Hf86T&m$O&j4@rNL;48+;zivyy>yo` zH~ANIVl<-JTpM>7LLzPt;EDSZjnQus@zs$eOs-)0m$@W~b_E7+(M11by?`f~a$P%1 zOtHvsAln+wWToTX|FZc>>Tb0jkBBIUc*ypvnB>|B(%}aTLJ1WGjeyVSM_sy0OOyqs zr`4~e5R96dqoW(T37!EQT*a&c)iP*s>DZuDu=}C#b zcTQmiDAH%`cXwHApAk!dNW&5)28KvBCKe_j#7NdfTc&CZKox&yp)j=9LODj0@;cFQ znTwj-3c09_s61n_u~jyfmWhZ%P^>;~XUD6n3ht(q0_#}FOH`>jA9tS9zJBf?48^>U zk7p1qW2h@ie&zybcja`gjHDXDaj}2@P}Sr-l>rYBP3H+ka5u+m&aRz0$ZriQxwI5D z1EEP@wpH2N*dXA|J$#sCQX#k6sNCDbE1T*LjS7z}|MUslSoW<<5B%9FxT^tH2lEGl z@F!bsjy1yG>*s;Chlf`sr4_NB=$ne8EOObVbR`lOh&&vQB}^s0eq3JkrBJaRCja;| zsDLUgD<%q=djdJLe0-n#M%Q4;V3d_J7iD&CXbleQ|h;>i6nq)6(J zP>@Mn`Wk?x8QeY)1616vUsKbaPtKix7y2S7hNB{7<*4K;&$;E0L-Kk_Q%jBXqJkzx zW^326mD#>xmb!fB`TPqrgw^6$fR2sbJbvK35%@Wqi%_0!QGu23&LlF=ayEa=i#oj0 z15l6)KXvlat#yJ#NDMaKqk}_9g-%y2kP~SN=892)guUJ zc0X%Dk0HR6?eR1>$37bXXPA~GI{$Du41GJrN;4-N7%Ild9g7hE8jBZ}?CUSlX_<^p zS34ARS4otK4?-Fiee+YF?(T=-neFWfkIJN$NeWDu@qnUXZwCis4L`8;DAF(wZRGmq zBSWK3FOcpA;_m@^eXkF#n=(Z%XrN;>uq)>SGjJv;Y9dnP%SvBU=K?gNLvhV1ME|#M zrdriWNFnxn9eBST zP)mep1658JM{KokhshjPcT=)WZdIa1a95Wr&%wQm*}36G!`XM?UN{+ek^Wz5OF1`= zvoy#fIhB;yMVpb)YowNtkdc;}T=m~R7;cTR1XUa{YEaqA1_*|{9KsXaHQFE^x-&{7 z`q=eGQW6##$(iWJ8!tGWO~^!k6A4q3N1SP@t`0zzj;rzuZ=d)}m%jzhIlrBm{8WetNN!_3sn}`s7G8;OMJH$io^YbbP<(kUkG8tJp2`#pcs^a7{EYC|5xca078?n;9mHg_lCA8%Oa z8Z_Id^J(5iT0*%OkbNJo0KyBWRD(l6q%&F#6^y@oDR#W;Y_Fd#8Y*;3dOQjOHz|#X zdt|GPJBFPhNg@5FZlAU$owkQ^oeEH>R4zh%OpF}{k${F?IVG&Vlz0K|FV-1<624n;z~sPG>pIi=Uwt!b5D1MjNptD!U@$9&Rpj$c4L` zFpTukmh@+Sjm&0G8AB%#s0Q`NxZGo}x_dG)Zrw3q9cGf9Uk-?<0+lk;X1{xtN^Tm7 zEC=SgPDR(>#-$afy)lj*5))>JX$nx9jlNM75L)OB(^Da@J`^Sp7nN2I!)qdz_Pkc? zFETA~vQm-H-qCum;ArO-g2+-c5;~Ek*UN@=Ep+fs4*$s}2Z(BBw^Sn0w*Pt&r3~g# zfok!VPNg_Be_8SYz@{m-Q_b+#SNMP-Ke46qTH!C_9u9yMq2GTMs;N_y1c1M}!%0eF9cd5c6}x?#Z~qM)GQMS8S(J2;$Z;xK)a z`lnZ`1dve)AT1OWL4AGK@@eiIpx70FAWjndCk9^#7UtvtUQg1nj#Q6AZpNoOb6|uB zmOb0^@`8kfEWXb0yqR4G~`7IW#^snh$Cu3He3%>9g zI*$zJ!;kHpgaa7m7PbC8$}wmu#RuI)$f=xIK%-Q#&wQ4s&=_@}(l0t$g$zV$ zP)4n)olqo2ycSC_2h}TeVpC8BO-l+3DLPuWpNqTgmXstVI;N|)Uy|VsSnFsISu^M4 zkP3PoCYB8$dKwX72J*`tRO9Z`9GvcYrT2FR>T}+WbNkRA{EFikn4zY8XMi5v#!vhJ zn6}|9+2PUQ`@4rYC$%34mF*D852Slp9STpaQ>+%up^r_+b=1|;^1M6-Kx(!iCpL;u*OcS>7DI-Sbl>^b zkS3Q;R!0W_FjxE~qbpZhDfy(6I6}-QD z{m5ICP^{lhmKzKB6SENlmgaoBBB@B3!L6a}aKiw+EQx7rYb%fpSBDoF9W8%Uma%Yo zl*!{XvasN}za`7;3%St#{>3zNCLf5SIddq9Szr>w?v43DKrjnc zmzNXbK_;Nqzrw<&q0jpuzW{opIA@%c5Ca7Mhk-31I3S=*qaMRn_4-o~!}|acU^^^L z9F!gy=$QBUm{a}k>zOH$Rym}jg28H-sk9YP^o!f0tgr%k$*@1~2OuG`yK_+0DlnGJ zu5;#wVY=QaCzbqAZ9J&RRVZs34{n}Wc_Il}}31(e1EnCWldzBP0L>Z*LnHNqgd zjR6U(!I)m}Pan1boFpMxpvYINo?`t^DBOSqC3RO$wIAg6t~)1+eN3N4C{WF5D0+Ie zB;Icy@e_FHD7c-22{D>mTJ)FCjl5(KA3kCUXte|7ki9@&;3Gje!{9tArs3h?#Pl3x zlZ&e9ikfD;i&=WeMDX>PF&X^c@o{!$YAha7tJlF{( zdKP z_^JROGX@ssX9wsQMDtz&l%f9&jhGNOkcoOnr8z?(Y2?M(aH+b8mrcei#?N)9aS7S! zz2$k=Z{|mX_)&ff6?Hbk%Tyeo8&Q(6l~v~`BB0ReR~o5YUj1$5w2nwjWG*y+8dRD6-A~;i*lP?Y8IAUcN0Tm7JdyDCcrFu5gR6= zp+U2(nr8)sGYAbu+$}8Nh|2ogpZWOUNQq*7QU9z8`lfRt#k>`2-})VGi$57vELR~M0b=$@dasX&Ri+Xs zB){A)KrVQgqQ%4_7Gn!OUwUw>5#?v3Sb zw$hfl6Sl*`OkSli>b+z6nS^R?^j2kZT^B9!BqIx zv?&(FZpkK+lEK_KJMa|C2L4f1v(-&ab2Un;=kN{hFD_n>DGDqL0A9l{cy)jYZqzDb zLU*I;^*Kz0;T4*1CTPx*DjK$HHSa?Qzxvmyea;jA-9x}JXWVZF5OSQEb>74Phk^- z^TCI<3FW{a?#U6(GgL{}c0Dpma@pUCjNx*Nc#$*=u!{4Pthij|>?Az!kju5IAzKJy zn7g%vcD~5&&#b9MBB54xfM2CagI%zyFrV?*tGRGw!LU%fx&X-)&gl(5ec@u2%Zgb> zKLM2?kq{Au@D1M(B%y3Y44vdIOQ_V664bD~mTk2=#Ote>pXae|Q=gfho}6A&BuN?Z z-t_eR*h}xw+4;qMHW$}7qLC0Ga6cCrd+`jxMSshZLgO>7`FYYz+3s$@r%Dam zPugOP=a#SOAayd&{t1I9KUygsU^*o*RLWa7QlKO~w?qjR79<%4=K@i|(rVzkT*wQSQ#p7=R zlQ03$pr*?;&&R`=3`T`PqU0`?A-K$zL^7|W{1tIN5>2>EV=;JL6!53K2K>+HC`HOC zq0uQ&;>U#K;mEOdUq+_c>C}JF4tx0La`LOXsgE{@s)1dHDTH#qH!ddlrsUj&W_Ubk@12l#=z2`TU8d>}qbrltUx1Vs5Ktz(9gk-flS}^f4 zoTYH2?R~Sr91>9`c>hA8+jaa?X(EC;^0ft2ps5P-?pP4I>I28YJ2-P{=1%>0xHKf| zRsoao(-@^ZDs0tQq3k-P0^d})*|=t>=0d#=Q?$lXeg}%6{oqY+W5V`#_Ye_~YJ<(< zFQ`cYK$4V(B2Zm9O*G13Lo$!S=_zfzI0%FpPQ*CBk_HCFN0mwhl4{IrgQF5K!Q(&B zro*vDf}wkOIsgWQbXEam+y}?uV2>#|RXR#)WO@x`SuR(tXJ^$BdCD1tgM&g4nh%Il z?yhGwxuJDZz|PQur;poV=ViBY^}YJHkO!Ovdc1Nkib!5pBr3X*8_gnDMoLgnRZ3-j zJCSnCy2Vj3Ri7H!RPdaT3?$D$5t1O_AeC}A3b!y-o!YdXsU8>(dW{n6EB3LY5*sDf zuv8xM%dcp0rv|)?MVxDXt)UFxb7&Y^qUS8rTu-DQ1OI2!rY^4Uqv(rKZ{=-f&rpBQ ze=&N>6&Yz{i`v4{oS#a72TBN^EhA9BrP$9V`b4ekt^daR0h06u-mhO1!oK_we^CLMYM~%7l1P6M4NZukpwdBSl? zfODu9(_^BUky0J<_BhfWKSJsQfDN6(cjeXHx27d{BNL}z2Mc!^*4X!Cx@Qs-@o?IB zuc44d@V!%SjpqbHxBSo&4@PiV;~$O{x6JGKuA#`lpY#8ODkDTXt847@Yh$&OdLvm| z^FD6>zWhHJlnNq@)qLFS>7u_st1JNMXA~Wc`VZiDrUC#z<4ro~KUqWHK>*A*A@cM7 znmY)ZIRxU6m8E=4O)YQ*&*#EgG8Y6(j)4}im6$%Hy zF%^(x`u}(AAAy+vBS(y=-g6om#S{3X2Jy>u*E6{P<+PFNyoM$jgJH+>Z@D(t$g4*4 zRh>R}2sAx4^7_7a!`r{XvT=N2J*mSy6V?%=K^~%MbXUZoe(x_rNDw zfEn6}jOIGWchj;CYT_hMp#({lBZF=zW%>BlWvw|@v-a&WT~N_OQHxLh9aodiptU~VE1GfKr*Rnc7W&VWu25@&b75zx9MwYoa803nX?QS-cvCG;KZ~)LQ zg#pbuS=++~!m59j7la^LLwrZ)eH`!!aBj}i?L;LkHI~oyPNf!18n2jw+ObN;CAyXb zW$=0RLug#(nM}qMrib0fSlLIfPNW-_CA>b5Ek20ixz_5TgUtx3#a2SD;vHSoOtN$1 z(F85X^|%_{*MoqHMAOq#w1~eM$NxN&W-t(umfwd5<&*1|6}&_uq}F%$Q>!se-=IbE zDAg9UxYsy0HZ(n+_+Z=CLA>rh+zbl#{3^L1$M=4jnRySEf^`tDmiX>+gA#LSS6~+k zd7k<anE*yXWq zrCFG^xyh2erVYDe9V=~XdV!5%c+Kj)x95Yj&-F@Mjf;iRmT>fS+nr?hS0DCm$UL*l zNE(sP7jr*-THep3>+*vN%tsLr;l15%&Fs~&_wFE}oUj>>td9FSTyIN4+a7t8z+ZcN zY?f-cd0r{mGpnem5b^ySUj68PvTIp_KV2Rk5wWwk$H++AKQeNY>~o8`r~W?}mbaNr zPM(rNy#Fn@6aq+paY=r&wpFwwfmy8&M0^v|>2Kw&lsQv8C-z!r9`(C#?C!t%Ou50j zGkSmfHazLGDY(Q3YMptwT=PR-zkWPC!d`1S8K#!rjyXMtA8Yj7J-&NWG-lQ6bb6{~KPj<=$iCSE+iPXKwHF@MC#kHe7l&Viql7YmazV37nCOVlI~bKJCt=Q$ zIWx#~czSlm=X})%l=G=&VRboN^z-oX>A-e5{xWpd4yC`+3i9l_bNBZ4E=GBQwDj2! z>~qstU*Xc-%*RV!i2L7*jVsu<WJOz4#9OZE#;U zv#y4)elHVIP4B0a=Jpmii^F?YVal|@aFh5e=79Z&G!zi83|L%33@6fGk1wLk$?aXsJ?oUNj@=7(|L9m7%ufNoXy$L*0w&K$(zimFWh!#x=zqH{5v}q*s)0b z-jEs4@UOE!!Zv3teK)!J)RE-z`u$Xs`_AHho(z9vvGZlL3;s@LMrFyTWgTKv+dVs+ zlIXZJ4LTfYIggEYpjysR4($!yV~!Qo=AD89dsWf6VSH=3t!-8o5gOWB?#$}yYI}P- z;1C5UwR}hRM+8n=C`=TO5)bShncS`q*^h_n$39nz#>(haUIiY1u>=iDQE|pm)O5a& zfN-_=QKq)FbrjLy&%MM80KE^g|y{W#&-)U?cMS>R&wLBpy* ztCRd$Fh22~+GATuxlM7o%oI!_su%j0*9r53jlPhOkc3Bb$dCnFlDY~&r3Hw6_dio> zxjtKTeJI_tA}H_tRAU8HBLvcz@4P?&01%5`oU5(>hu|0pP>TVEqlp>cJIbiD%>Q1W zIHgOBr`+k#C(!$xApTRjPXRSQ!%{g_-YKeWU*(#;&w882P$&7gUOHW8+eW!e z)DZ>d#Xnd#^mc#iQ?TxH*S&JDWBCW~y_Cnl zLQDx%t4HWIEI(V2b}<#L?#N=0F`}In32N?&2k*1CGmiNAp z8#yPp{iQm1h?!92Wg0yU=$?a7!->A79^O7aj0$Q?=xC{di65`EZ4P#bcfLs=8LsZ{ zw|H(x<;KO0wqL!-U%u1_71U1f5lM9m`ZUc9PO#E&0h0F>+dA6dzW*qYFC7>llsj%* zQ2DawMmoJX9|hC;GP=IOek;E|IIzg584bFO8j-tJEIVPvi3Aw1+I2>Om6a%Gf#2k_ z;t(A|(&Tfy{*Y;b5RH^U6?odA%h32@s-LK3-89>@NB?l)adoY`xR^#}3n-Gg7JaQX z|DQscFLahHWYD)g`5GrKB=bH2Xk8VZ55&Qe^8X2X0+7JvrNbT-|2I13#?>^_ek4bI+4ooS&l5Wj z39iQ6cO(fELQBswpVepa7Y&nw`IR3~hEUNq0D{{65PB1@w!WTY#(7BZ7wj_Q@xyfp zPlXV5I8&9F+csG}i2Cbc1;U}D?2B`Bf4yl^fuWA~tNrW!>VO0c^^Ajg0t~er z%*bEe>|}>7Og(sUa;eYd({8OGd#LCC;X@LmUIb34g;h3<#6M#@$_TNmwn+f3r>A#% z`i>`4&e`1EhrpT-2onJ6BqQ+NaUSowoNmh^O)i~yYiesYB#HS)9=G}w-DQE80N@m} zTWO=KOc($Hg2(OW;JI_`ij_v%SC4x|$3Ox6INCPqx^A=wDj~!y^d;=(=VPV zM6(hU?1q0lWh(F?+Cv==dowWp!c#(}h=_=H*Ye(8wZa0KvMa zS1epRQLnOr&BWKAxsO?(Of_nm=>Ik4%Rlo+;N=1|;ex zC+U$bhu*UuQc9FwWezdXWLU0m{H7FV$O$bP9qhW;9oeK53Cet`4Ja4nUKH_q?l&t8{p>LpB^l>53`wgJVq1OxjNg-wZ23ui1P2MZNjNu&@AIDKKx_`JI9vW3qQf_N$naNA-3e3{oSLw?mL80`5 zD}fq=_69jvdwm}#oZyY+LIZAE+ING!NF6?p(Py)TOEeo@wZG)lSG6Dk#gG_YfWvBM z*Bv99MBDcB^Jjq4M6cNjclmK(;FVYlpi=t$eES+uc46Zng=*XVi*(LmJiCo)c*{mZke<1@x{(Oq@@STzzLFMI#3>Oj8X7wK{o1$eP9kFY zZ0_y#Lv&1B2+|gt-8evD>E`Onpfx2sU~#e0P}kKJ^BBVI4;p~ih;p8!I+LQ(%Nm~(`1tMZ(4uDT$Q4M_ETe5>4#v{=wd>%Ca8Z z^73+NITVyUZy4T`465@k6#IY^`)V*<7x3!!9HyDSLev%dq1%;Ip) z&MrWgj%M32U&8itFVbxK873jJ11Vkr6Ea3kGCI3cwiZzO5kWP}2A!HWRkvp7D^v;C z%etshnPlq;Dky1%$wV64w`h!C?r!^J@zYUr)4tn0+XSRbZyb-kdwli0ujW4iNj5Dm z_d$FV38Y&1pFNl7WWfML=U4;X5^_{fcl- zX(`XER|wCq2hXN;baic8LR+tU29>qIq?btzX0nR!-l@+tq`` zl761wlEq(?Py?6T#AbZt=1W@{Mi{vJ0^jRtAANnpNJC@ed4t*IW%H^H2vyRQH(-`1 zxI0gnVw|OD{_^6b@Ic2PUH`0Qa&)X=WgRCy5O~;NeLYA^mI27lAf$7Yb(dqgW&k1x zu8|THX+WI-MfyGkkQt=BJscQyop{9Dvs5RwgjN9dLWVuVtYh{#B(f5%bL{X;P+it8Hj+G~1owm#bOi>qgOwMo4%= zmgc!i_ay5Qi_1#N{WHEKR#NIY2#tXa=$Of`yKAHIAA za)&(JJ&d|C*ft`5vYAYJUt|N}RDYBe&&TNJf#P$TxQ*n-QyAl@-<>p+IzanahS6ep z;JdfBif&gYug#>^*Nw!jzOQl8^EGTHCW8$GutCpZpIh?tG1msO*d3*KA0F&2*BQ4M z#o{PM(Ic*W zd2O1hWE$SV(J%cB zIcs;YcnHCgQJHSuGjbazD)&<55_qBV@@$@#nz&m&`d;425_w%ZRT?fg<@Xpgp|Rea zF-1)SMV5mlf?>zKqw>G32IUXF&f#4y&^DS^n@rwOG8(il5-KWEr<*3`sxy;6uxPCi z!ovVfxU=C{;|BYcxjHA_af2n@r4<6u8C?C3I3=5llk;;1Mn*JnU-tXUpCmaGlQ!N5 zLEZKm8ZFrgDOdA@Ov2lSM%diqHuGPzUP?iRKp&vqcUf23EVZH;N|S$)p^HoFqa$7N zpVKSR!$2mwf#Cr!xh{(gxfxbUz!rA$@*FR=pcFtnhu6Hm=$nw*Kq|nm6jYse! zl-LX;#mHBdW{%Yea|6oL?FUc{*?$T9ze9-#ZI=FeB99*An_g_)wG@0z}fFd{px z{YALzq(G}W$MP>>N6WS z*soxq`PEmhyXV1I5bMhd*(hZ9dtR+o-US6h6HQys?I`jD zR76Ug$IsiWTvksAwa*E$NiHa|6Vq=(9}twZhGOhNznBmL%Q^(u$o*%3bHEuO;0tmf znhoX9wiOunDr^b;=PB&m1k8FJZv-5GQ}*Y>1<<^+S!rrf{=x{{fZ>{)4GHsaOhO2t z2?KxwabW$(_=CJXu4{+QFKP($I{c8Y_7Eb@hj=Dd?1PQ3$}SmvQZ9c10*Y10x!{f{ z$)o?QU_dvM^!to$)db3u<^6xj@CHbbm)FKW|2x`%8t4tL@EA}(>L1S^@P8pE1a{eB z`f?}EzY>$a&;34r?7T{`Kta)NHga&NDs0cGtsTt0_*{~5)cyIrN8#`rga4?NjFhOv z=*kPTvqdutAz##s{O3L=fhIsH71h^1MpUhWLi_m6;{-ooM zMG>pBk(W~H4h^8-ezL3$@X$_9zH3c*wF?LUHMwzKAR2H)!kFUjITIJ|o9cUbq(wJy z_Swx>3Gg$>M!8*-CN@oPCNDacnU*LI1_8bWyDOAqHNEeJfImzPSAW@QR4HyKz24YkFFmkco{94F&h;x2_UH~Q^-MVCS-Wzin5BM{p0K|g2F*& zUHV&Rp31(y5I}Tdt{vllW`kSTM+1rMHSWqbGS2q)D|vT!@kMg5&{vNKORnM4Z-ML{ z?S**(fb=#Mq*Mf?qrZ^~d0g^54qsmJUcUfGMdf{qemukOxZ3I)>k?i+Z*l2|TJSm9 zj~iE-r0DrpsoRFoA05=&3xY+iud5?ZDc9zZloa1X3JMDw@DFb9`l9N-5Sm<2W!s`e zcs6N{Nr(x^{j@)RP1aXapE2-#l7|5v%+2Ndaf8?!-DZz?^8Ea3b|gLIKC?0af~>`8ucIy0wTFJ%oiI*eO!^7ix`&fSwhj#5Vs~ zT=dU@#7}@4*+PSj-~4i;x2~r9a|$;gZeb+S3Z%*Qro?4HqY)Ee1gO0U4ILh4ppp81 z(qO1J+a|q#6!l^YoY^uyFRKd71&JB})K*EUsPJ@!Y~EDZWFG-?eho*9+*cQ*<-y>u zOD)XE*EC|9Q~r@~`VY%QKz2t(1&~p1W@Z8`0zCn?{*F>@f{V+`?%8TP!xg_u`{njF zg9koljB*;X|DrB4#m#Qg&NYm0=Y3Jk8FNASkCv0<$zouF~7&b?j~jP&i;P)A@L@lJX<}w z9zWW#v%8BK{1N|Xxh0waJSQy)&^g|ocR-JJbd%^ z@Qlw1IK2`DDK(0K7a93^V4yaq{g*v9FZ7+`V@{mNKI5sGncJHiEDWqJp$|caa1am> z`gfb3=9_sYVS}MZvpGj?ViHW;NkLbq)P1X4BUym{JU1sN_FNa>E+v0XQ=!`DfnRhB z2v7iiXg_luU4T6z>y=#m>C<5Uh(x5@+NbA}O$EUl`C?OW!|Lk)>L3!uxb|pD2;7CTFrJ>9+Ign9b94mw2AhFw%4^I!DVEccTKIc96#OVN z96HbMLe>0kR~BPmo&$Kvw@(!P#NNj0gNrZ=pv?)RkEj$ES0xHJFC|3)k9Ss+LJBDB zA!%zbh(%8`=6}g0qmi1;kJQ1TkpdRCT?S|k7aNf(jr>dZt;cN@g^krUW#3qlaRYnHbPPp$>x0!fiCs=wRmkmncH-_I8GHTh629%qXa8!tLi1cC6=wHY7Xj>!q) z31`Q8!TC6D!~iY?MSl#3D3)#i4{_JyC3IyNRn;jCtKI#5r?qj@J0gJwJ}3ZCm7s&t;B~k^Ym%zeQsuag5`Gu z?eUVZWk8~NRsr8*0j)cFia=O6c6oXE9rbHddOqzJLkB=@`VW18q`;S#{ep&qdmno- zluV}@1}Ou5TBF%1N=RQk?b6Ae<(`)S&9y2LN?%i6ztw;xweVQOMhvL%e4Tla7P%(d zn=?6yVX~Va2uiv8B7>fEBrIsyKW5F;5n%m%aP;*I7{%w3(@lv`Ow~#H#VgoI{}hn_ z=lT0ZWD8-6y@$ObLHE1Bve!sLs{(BV;`QY%IorX>~q_Y`&O$cBy4)yBw!kj@~M4i zgYk!e*X6rf-&KYODxw4+7nOW!*_Y4u3M+bM02RpF=NjD-2o|nVrx>7&BbWVlXu$?tJs&mM3dXX_8`6VW#R2eU* z>ExCgNlp3J!5KzWqJ|I%m~U>hKKRc7em?4GNkr)5QaeR`KXp^gAMS^V$){ZWr$6Z@ z2Q!Q736-TBXfkONzDiFqAZxzZwd8$}1~P3T;SF-v7dPqJK%nFb-Rx<3vw9GanZ!Kx zKH@NkXV1dXh|Fb#jF^Ogh~MRl_I6|@uC;_&PjhoiH6)=m_%(6xL0^5jHXukC1YdKG z%@j&CYvVWIy6gotC=Li(k7jq!5YXyubv|O!Qhx;KuK6p&8-%E zs%a3|AQV(*eG?9+vRPf=zX=5~IcA@p1D3CWW4K}ow^m!Dr z)}~VeF@H<+t%ZjOsUg5im;hXv)sNWQ7DAOwjI_fe0|7`0shfdpv zUGNoo_RYm>;YG(q{-?u(qXWuyHE)Cy0;n22rc%PZ^)Re#LiUFBT)6St2qh!9YX%mK~iQ_Hlp{IDAy zK2J~OG`7!BwysZm7pdN0Vn(i9{rGVd^e`vyvlq|QcDE#jpV}PX9x-xoczm&3iP3G@ zpmi!*(Sl|Rwu^QTd9W)w9Ew-v$!+W+V;xj>C@6&*`;l*(K^5hk6+8M!f@D+b(>%ek zPTo*mJ=WI82m8CAL5kLbi3Tn zhK}>IP6?Lx6CB@ml@f+hOJr)hnso*YaQYp!-}+WSiSLFCiy8 zrC2t4Jd8}MFJB)k-%3{MXk0HBI2~7HluWJTWos=Wen+w|s-MU1cuPJ#?tPy=)10s? zamN?Mnv%-7Zn~JC`Uc7}^$KlSzwt72IqV+xzIUb_&m7ao_3$Royq(L!$GrJ8w*~F1 zM}ubzk4;O*f@MEB*{Br?L1h33lnby`5M=Q-@eY z)7f_GOL4orr_rBwl<1Ruwo=W1ko8@CTNw3z*sFTFPb>)+T81Q6AMPPsRywcAtrcV}C}+SDEiGVAxSZJ=9l z-Tut_!*`i}rM81(upmBiPAB~ zhYTXE&xdNndEGrifRT$jq}x1g(Mgz{Am8Dx5a{>MWJYNcZ&_w-9FonG2>CvPfP{ua zO_4LWi*GG*r#^NzRu2F59f^aPQ9sY$@Y%Ne_`|2gM>=XbxZ>f^O_ zv_cDc%%{sdHRCsdigPU%*;QScN^wd^SW%yBbyy&V$QRk~B=h^2&J@Txmlnr6-+76e zNxYZOU>}>ed>Chz^(u&|`lq_LBrowQE7{ZE6wKSHX-h9{dq&KK%2aFR)j3WMweC{> z?wxO^GbCCC2TD}d@?3|*B%e&ukN)OMtKDWjaA>o?@rp3z_4@o=N9D9&z30_M3rf4g z}N)7re2*U_FSH-+gtl*_v2H}XI84!&Xo~zR2&iN@LBXk5=Lj; zMj?au%+)o)0{aF&?SZ+8K;|Z6{SMNo1?)s}D zqzo+2tQ+#IsE@=?1}C$Bnwqq0wFy` zLs?xmaPDO~uf0>Dz5Vyt-j_Q_>Y1N8Xif@{BBMZ-O@68G%Gz5y6w*Xj9<$W{y$fZ8 z2;vGD!3^7h?$Uq9T0sq>y0VNlPybJ6MV|Mxi*U35Ie_Mq@2vWC>KN1S#VeEpX{4W% zfk*!P*-B6qI-uJh+QI(4EI%p$?J3Ab{#gwJuSmd&@6CSxdFm93f%a&MdjC9hrkJR= z13auZ;=d=KMg?duLD~3EcWMwqK&F|v_hi4XhXJ&QIZ*QYPj@td?$(S%c>lQ`0?;19 z<|NLa?vw%D)ijsr_;bBLpuIpxtBBugg)l2XG622w=w^lI5;2T*~OH0QuaTKkD5tg zeD-n}TQB4|p7`OXWUKbn3!X^wh|nMFHaS;o zmE7$gwXt0DUxuEx!9XgZr_Xu(I@3a2?6IBBa z7%kGmIg&qBEHW81bmwr$&XI<{@wR>$es9ox2Tf9bW>-tXG`eCOZ!dFFFHSJkYl zdyK-Evua?5$;pVqKw&}w006*ef3|%0s4u347d&U!lK=w1$+u81M@O<-0-2yCu$54P2Sz+04GI2cP0ypOiK1<6Jl@HDxA}yq|UKeeLTL+1Rvw_Fpgl07`ZZ@HhOIR!SLV_4unLBo*KXCQ-K-Q z2+Z>6bCBSE;`T#>y7AjfWQ@lT|CG=*)BY7Wu5g3ag4n8?RueLF29m7s`J>pkU~XtG zoN+e+zgWOZ9$zXlPz}alsQs+Pu4w!yU&?teH%bVTDq<68N6n9%m@h8|rBKq}FzKPF z)h*z5;JYaldz^$PJ}(oGQXF1o+ECIE+@XIXq)=rbKMw!KQfN=>6{sJ7c&p($ki0;q z!eIa#FTM%`YDu_be$qor8a|oOUbE-I+Q#Niv<(V(e?T-bJ}`Iya1tgqLh%hE|3x-AkdHtyfSce! z(BB@Qxd1o1A!-BjfQ6$T(0fF6J&K`@{A|Dd9u=O+r(>Iiu0R-jeOwQFM*^-7x+zaT zAK&m_v-J5%1%}1Ixf$ocX~OvX`3%SzQq3Z;wmMr_aRF;SYJY7>tvTA^Zk|KRLY@Z> z1!4ul_GXFq8Vt+IM+4%>@v5aoG~hY3XO=!m5dxBoKqMs6UDHNxsqdT9E2hF3Qk zV*NCkyd-dj?~KbCrQIK`fA8y;)sI<=de-s{?!21pW}W4TMg>>)`y9eY-{#iH2xHoz z((c^uo$z!D9eH9$0iZ{nYr0TsT-u~w51qen55hIJD7{vumKI>Fmo9Qm4J*UVC0PFI zO;*MuH~!UBUW9BVh1@N>A*sSx_Bco-0^S5%|EsDR_?jm{k1aQ0^UndIVi!h%i2bI3r?|VLqWy zu+{0*3{XQ;lXQzp$zVWeZZKvCP0F!jlY@ z)o2zV7N0Q}{O%3#>Ci+UY>aPZ#+&#&3LRfbfuS8w(KBD-D&=Jod zbwFVCsR;t;ljH=*$*jn*Ni%~c70|YE;>6}iG?2Kr>Qkl02xZ7Nz^S468OmD9?v=ebMVN1tF}MwtlTmK06hm`j??H!wAX zG{9Lnv52xnW9nKrX#{CxttdC9oFM;@{E_@3bWeEia364@I_JScj;Vra1yemp9R(Fd zgxSpe!g8EGkuIO!njXR8Vy-iHwm@jEZVqaG-FRSWJCA8rMVMC{n=)C}gVY<;cbLG9l^sakI^8NC zNFR8ilA;^G14&6EUINUd)gXXCmA(%mv5A>YD}gZY8z=AdTj=q?3ywS z6ptMIqM1%M&#-RmX&Q%ZbgUn3sHkl-h*@b{g+6CpYL3B=yuv2HlxNvV4k!m6&Fjt^ z+>R@#HVicUY1pwl8iz`xC_;d|w zWHw-SJ$He+_3@SQQ}K(TsMAd4W6K`lX2aO@}&5HN7sxb6{#{oD5C>|Xk~`b>T4ead~iy^A5d(X|n=Ld?

<8Y3#)af{JWxXR`6?pZp-s`*h+fRW=tww<2)E1&?9L)*&us2%g)Q3lmpi~a zDTAAmwc@d)%p~Px(Xy2?o;k>=4TNQeXTN_k4e7{y7eggwEsRw_RBS4_i-E>$cGp^T zZN4ViwUw#OmSfxA5ef)nM~abE$m!2=qECuVPy9u756v2`IdCq!Tc})of4$Vs(vIfo z^Fiqy(=A#na7DdN<|XXz5-dA75}7U4HFh(mL!jBB^F&F-Y^k{LdRBUdvkc5G&hFtj z+*;NuDx{HbC4XzQI~awXa+Y(exc{8HN8 zoSVPZH`iYV4$T5B97HpFR!o!rP)Xi$)O`Hpv~gFAQOtBO;mVxA;y6leq_R?3ynIl9 zIC+td#Ate`b+UHW+J(l6QbX>|NoTIJ>sj#Zc;NDZcq1ttrV&=Sw*Yhrnt{3l9R#h3 z7NyZ(Gu^D5F7vUB`_kQwrx}po7Tu5iGebVW)%ecchU6(QXo_wj5*P3r8 zd^o~Db*jRkl2q;9=00;0v0uHPu^_i7w&+wDrM6l-VX$8JLwCWnx=S8G?q~B%t%Lmj zE6f{=D(j1F<+@QrnbmB`c<*>~bD&jOMO``Nro@^??fr36zxAPy;fG4cx}L}KLk*`8 zCx`Rkg;n)KY4wJ2i={b?22s`VGPk$OYIZj_Oe1Wg=EKPa%_{hV_#^jb&ack{zbjyA z@HB8sICb1|E-79ujgUF%iqne8%1bjb8#7)+=fbk-w(BeTp#sk6G~5cdYDdpw$dj;V zGB6p`tTSE`o_PHAg;5dBQ$BJWJu_aw= z_p*Geie7zNU3YP~X#b^Y_judE{s{RfiD$+|?ql{;djhx0`{3N-g6)I+RJw1!w(F7A zK7l5ClkIt&{+amD@Rl}Qe`1qzVaVOXZ9O@}ed$i%ecjr;Cp*qdrMuG3b(iv~OXFqu z`b$S#2c});P2_Vty1Cjml9tEi}F}cpW@Pv`%YW7U3;AaZ916ycwpKs>%%QT4xf2f~7v2A@_3$0TEwf z$z-E@gVCQztH;kfbkRs;KR@Zh&WOV`_m+OHA(F+D3<(dM2n3N%3(*08>jMCu^`QII zc%8tYjbQ~vCtx@u8uxh2_Bi7MUWU)h0bWB7+HxPPJ7cie0oRCp#B6s#fWivVEj2Zru~UV;jclSmfP{m) zJ!s>ZaLxmsrMK4A{VW}Y)s7#){KMGpnt+}dFsyg2`?%$2vo0kA174pr?T!P(5$SGB zVc{wEp`{E&i4+uh4lu_jXGyoty+3(l{u}DskGwV426kK{%7qNIr#1htfoMfS*m<%K z^;n`Wu?7EdaglV9jfpn$`|T?PMktGO8a(Wi#|s4q7XpA+{eFH9!#(wIN`Bb^zO$bL z-W#jBImzBh82L6cy^&{E7kzYeB+tLPut9tN6cH5`x3shr>n4}k(9k3)I&^g`VA3S^ z&)hR#cK_JaCWf#BGH>MyY z6d)uby3as@C9AMW++1C(Kw84+!kaKf*uwqsju@RifNZ!kIb?*6M9O{dHKV*iwjSb6 zLX8m!A|&)HF58XzYq`2xM0|nF;09|J7E~Zhx;;j^JiZx(Mnps)$yG^3mgjb2D@EcverEN*s&vTaQg8hhpK)He3J7s|HU^G>f))jHq1D6 z799g{=TQhVtryHE-4=8Ud*ZFJ(9UUa$t#W!9QMfoighF zXW(Dr!hmmw37B_a=p&n>g@Q%$8Jk8tyK6`cxkvP!P-}%>BAwyt!ty+ktRiDCs!1jd zyF2i7+_a!KB7hcrt+0?dtgj!5Z+Y7YI(3-s$#K87u}E2CRfGK#|7hs9F%!neOZh8_ zzIeBT}`NiAn3ncUMwT zgn)Na2ECV!4cx+8Qox0rganu!_?nfFUoY;`lP{?`#Poz*0yY}03XF_1ML0&g_lv6H zii!yq$T~$0W=pJzWvnKPpizG}n9nA9CiTbuIsPQ^Hnd8Oz& zry--E4ENMAS{pJSh_UKkuh+pg1)RgcB8XwZVyofi+^+t4veALikbp|y;p1ZzVq?G) zcK5a+&Et{IrIk9=Xo;VFdj%Quf#M;fe$NG>Q*fw|#Vj_DD19rgb;y6?wVD!#NHEc` z81c1O&FFu9?Ep+qz?k2YmzPIG!J&YGf$2+j84*<4P+g@n$BRjbB(X=t4Ph^V+wTjk zg!@K8tw2l({KK?gTuVO@+vk@$$HSeND-R!iJJB(AB6)j*`pG59m{5r&eS?{}S1<9!K(|Fm@9akiVm$b^=t(@UXZm1xycm;6-d^nv0kqa6e?|x7xT_YF2NHM zGSm)@fb~4ofqC!1>6kpqGXVK>a5sF`>WYd?W5;TA(siZ$dBLE2{+FP85yuT$-RKjo z@LuJ%r9+&C%ReQu)yTvNAn^q)Ei)Em9VB7kyKpAbyLtZF`p^#{z9GWv%<-vp0b<5X z?iGong7tr|Uegmgh!wx-F)vS`(nc4nCnVZa6jPmFC#%qd&WIScBu4Dz=Rl#fwBTt+ zVX9uU|9zOl>SrL*ZH-PkBqTixao?1|%q(9D$J|*{t{bl^BsnOWI{~McREd-TG+TH5 zB$$vCxus}u3mosLnvea?UvS5akHbv7(kf@HnfYh8x`GJQ2Pn8&0qjZ82)pSEDT3|y zH_64;CErqAobx5`m%IqnA|apClJjjhy1+(he-ubP_p?h&VdWnN4BA*((Li})w?aXd zQgWF7=9gsL)utb?voN<-x^K)6ERf^BlFk9qu+C`s!fbA1Gu>4CA&zuQLPaIHsu(BwL6${k zGR|nzdXtH%SHte(-#-gDs%JgJ&=FM)SiM$|8;TI#tyKI48ToIio7{lxB735Bu{yqj_6B(#mA+BR1C@=>YIvHqOrNd3EFhU@lW-;N|!mD&r*?d z;mXR&9JaOqmwy_TBr9rb1D`7~djoUh$7V4CYQ!z$vs=M!a=yaDjzczL`D&IFdXN1e z7Mm@WQLb}-6H3Asem>hA_8qj(-);zEkL@Q!fwp4ugLxfvu>-)&gn;ifi*0g#ioD`R z@q;Mo8>;}eW>-G&Bxvi z-p-FvK$JL$PRt!8h6y(RTA#2NRVQ@1Iu_hiF|Yutbor@+yKZbM69GO2+t8$P zyzV(Ftm_iXVetCZpg#h(1!4D%FZUU4sKI)*c~sWtjff(fQ9C{NhnQ5T;y88CBFbn= z1-vJ+0Bz7k;1dYTP`b;o*7NghSL}0&R=xlQ=!2?`n{w0TT3PSY4VDeO1?G`0U(dTp zDn{I0`@@Q*SiNSlj!PC)77M&9QJ(O`@C4v6c|y+T3+VGm$_Lz7ba;aftRk|fc)X7L zwQ*S`2Layek#XHsU64ig$oxHID-xdtVPb6J#U>qH=TrQ{r9?U%3d(R85}DK3*ERRF zTyHwu6JX3}I)ROn>*3yJ{PBkQd=5>gAJiq!g27s)MKH@BUK*r_RXV-g=#kRy|Ipm* zGD3sbp&lL%71Yc1nv^i^hs1+rb~+e^wU_1ceWWkviI|zg-a;V5f1sGtrc1lYdX=>6 zZJa4SpegouVM!4spnA)dLYTN@U%pcLo*z6m(9iryuttHzBUxJOC1b*~Z&r6XAZ z1Hp69*XENbRz1=mYSpb4$y$!%^X1P3*M@ERaSv&xIiSnwR7m_XUt1$p*V}wel2|QX ze`sK!BgQDF|Kt6MWlcejb@}~P3^CMs%jbs2d|_ldq77xYi(~>rld)ZJ&t;fUaapT4 zZJSdbwSO9(^wg4ygQzs~)7?S$|XZWvI&Wx)Jum}oRidK&WqmTE)pg#iWBh}RJeQj>i1POLN1BH1XA z(+YsSjG2Uf6+CmEPvXI-%ufvTnM7I8#5ne;W5Lk+`1tQEgpknsO-Ycex~YmVX03vz zg3lVj9P#uh#ehfzu~T9DMvZy5>Xm79OAc{OVF!w}yqqtAz@UW1zR1+{Ws_6N$`OzoRYD*Lo-Fs7ed z%GhiZ%lCzfTwJ~!afPE*enqnM3j2pG0pwM=C=N9^(AUI7qr?=@SvJ{GfhZUjNKTiF z`2Er9lu^1a?0#DHMh;@zyBcy@N)^1wn{Wvd$NrF8Xh>pl&{wepvccJQN)A$q5z6S{ zc~$K)9$aRR{N%6$y>J#!9#ra7x}j)QDtjUZF<|dG=eXRfj=_Qc5rOv4)~QfsBDzCn zm`b>nT8e~{-v)2lrGVQW#AW)aTyAxhAgz{GR>?uoXkzmGd7ZrJZ}amD#~%5%3gYqz zE9*RW1{ujFcz>yfk#HbDnbAZHEjA&~^QPObxKUwYO;sZtPeDD(<{>(c=nG^|!7$X> zLj1Gz(9fOIQ=|zj$&g-v7^Ncz7IFYcM@PMU#?*LjbI*tv;K8p6!u0;-SZ~6I1!oCS zSnXVCOWJu>vUa9?h2Rth?Z@j62tm#i-~&GC>4 zTIRfL0WQ{KXM{A`Ew4Z=bwdmTVd-mgKJP#IxOKRdW-X_e>H>}@Xv*-xh7h;$e(jCG z$3J&?wQer^-iqMmd9Zvq=M&%t90o`Nlc3M>-Rs3E6<@f!{gnRYt;=G zGH_V4w^y>eT>z<`f`kMSAZ#S~ducAd?u=ftkRs*FXOb$mykkog7Hc`=V)3|UAr22& zj{6bdZ~&wR(8$L} z)%%M}!{_6=M~sn?i5QfaOyo;)ZAvPf@8iLE#(t8jZi+F!K*|=Ypny6A&^2)bRR9Qx zc}BXXQO}!O`EzMSMM{dUx9~#ln=FrA*BEfvD5!h~2yudjmxY4Zh)Rt*w3NqB^`37e z%G~s$hInc*Wd_F~bRyUIgEE4SmqbRLNE@1^}tYh;O(if$&;8_D1YPz0y z?PDQ<`@RN>jD`!7Q0D|e@*{N25$`E4IQo))JlRX}6kL!u zWX@~HMRw%G^`3+z2677YjaMhqqb$71d}S*bc;tnCP3;S$Yyo)rinjH@(g)Z?5o4lXXZZ8As#hkHxI=?dc|erP?k} z)^8d-BO9HDA!kdSie6mEs?@Q07C0~FCXW+sSo0+%rKNJXAA5op62gv@h{DMwhmA-6 z>TGYW>49x_(|AEf#*{1Ypg_ISU{Kqf!6323`yy;Rw=u>JkTDvm zMrQ7%&lJ?ub2#0&o>$)HeqHeI5>3{dfW68D!>+8=pR~s_{G=LP- zj{PEAM%$2g0R~(b>xMijT5A~qZZW>E+} z4O6B_3Pgat@pXXPom-tP@tARV;lXZdBi@f}eK<)GG~AX`Yp#|;2`jXdq=Ohf6v^K=j$!>bq~P83Nypx4-&%lG6adq)_1!?WdKqP(W#E8pc_k6^`Ig}rI7u|xS6U%-MQ5r7F*1pC@!`?O+Cur(!N!rI>Lpodr4 z+cOc6Au02vmf*v}0Og#r9VZ_8fB%klaBjZ$M!BAI_V%+QwF{b7S_<;V_khYHE=$LJY zWUHgO3(JLr@v^#TIahg7qMkX3gQ&GWBYUl`8f+QwtoJq$K}vi^^nzl#C*%SVhAY+W2Z z_t5|?o>t3|@<9>G)>pVNdJ0&#_ZMr5Sd2tkl9M%T7(*P7iCY9Wy#Gsa80(zrU3c=QsROw;r6 z{<@t}3s1^5Y>_X_3v}Vy@Tr%E3}*(dkF9EGJ1)gOexGwjbdiSmxh zHX!S}5a7=VDu3}J@gQWY>HTmz+a2sy>!e6n@Oo$D)OP_&O9{=<$_ z*M<`t7!TaStMhs3F;P#;?HFkOO72tvTpDy%Axx^$tbAb`sc!vAA}@>O=X%@}R4ooM zJu^3{m!}(?1Ag9EA&=|Fw=vC@^oH;YMP`7*OvC$+*BvP{*i8HQyyqHfl>00l@ZP`*s zj-NA+lMjGIFIiGScoPvzW?O^%qkQQ6-(rI~)p3GqLYt={8(vEjIV~Bo11n&>xn;SY z6Wf1UL{V#cSG^QnO%0TvP^JMhQNO^SZ*+jKqOTO1SaSY1>sB!?)1H7!nf?}A!C1&g#$)`;kg>w4(wi64JO8yiYe3&TT=CclQHcA zqH_GHMpM{LzjWl6RMk@lev{jkHT09R5zo|*m>}xhe_!->HH@EqkS{FwQ|^?+ka@b2 z^nmyRd=J5vd+!UDZF^ZS)esg`I>&__zP`yk7oT9HeVpd*f}fR=o=d@Uy~5U(l<2Kp zDl(4zCUklfVg13P16X+OCm0Kr&a>Aq?u+Z)C4UsyO)hqBXcKOi%^xZ836S{M$abyc zm%qj}xA8jwR?MO1v&&IqoX%$J?rx--^_}Yqu0xH)bEp%1z}nj!Nx#R@wJxUNafbL1D|fPi*@k2 z-oLxqo~oSPG(Z-WWM+9pBmg5e5El*@74fLEq!0a^=q4eTN^w9`%ZNRV# z)94V7hG_9AC+Yi&NHaf7D{cDw@Bx9Ths0q|q{94gbx&CRf4zPsfGP(Oe~aqUsoe{xF{NB`;r6>q*d!VwdU@EP5p2C@Tvx=47V>Kg0|V z#F2Tc6%B$twzw3EQ>xcDNuxo5enQC!7RE>X{fG~n1sS_dZ9yMm(n;>|jqH~_;RC<+ zyzQ~2ynsLz1n4`2nXkB3%RMccUa8l%|2M&Ju=5_rMJe*mg&di)ZW(@izNQ-OHz$^e zHP$`B800mq>BVw7K4Nler0=L^#RGjeY)uYiR2XW?JBZD?tX5gacP!GhKRw*2whMCd*?;kV3 z?y>mNVd2wwFRPX6kI@zwrC$15S^}OjzMjhZlIv1tU_et^Q;|`ofw8eySyM4VoHit6 zB*XbsXl3H)jVu%tkSnUP6&E;q6)R}WnZ}B~mji84ARC@J{l{z#(92;(3=4%@K!wQN&II!a)+zr6|jgL$6U8-wiD{s0X6en7CQYGrrp_ z;8cm(K>HaY++4d1xy5u+yuu|8PvCHVS^D$iW3e-Ldmjbs>jaHGMdA;hy8#HH|5kh~ zK?tFsB076<9R!8}D%9qK6U_NZCB7bRwmEE$=`9QxF!B<|8CboY-NF|YuX@jO*8pVw zo$F#@Vst`!Jk>$AP!CN-4uu|Z#5B2<=92)~!gD{9AX1o(LPn@yc+kV=8Qd`1Za>+j z=oC!Ml|b`la$U)8Q8o&+gO3%~)xv8MtD3ZEv03kL*9+7u@GZ3nS7Kmd0K*_)0kE0Y z_8=diTv?eFA8RaS{+qxW0O?rr_}tEw3N8crd{QuwppTer@G1Oqg<OqIClpJeOo=FUt1sVIVD(akNi63HQvRL^DXNYRK~LuQP&j;! z;Ukh7aw#qoTul>sRaA^>!VZ+%EawzX>ix_GE(uXQbpTnxfjX_}Y=R2;!13$OtS&6Y z6?i5{bObU@h<%!(^9OPk24&#-rU37&_@hWk1vlhaIl({+%wPfb_Io^WmYmG9_;qcU zj1|ySrP_2u)IPm6ve>46qGrTL!}o&d0+e3?C^@qT+17_P>X}7PxRjLbj}i(3QHd5A zSGcaf@n&y2l2cTVWYoqc9Ts|8GbcW|H21f9fLGAI*8p+`pZ9+6CdR5#11U(hfXn<` ztkka9i;6Iy&QZhZeXa6K!xrDz=qR#q7`%z;$vlVm*0ii$od=6e7>jgT2A86uipj&h zq!;ZR&1!QV#DtE*d z-KyZl#TK@xUu2EKjJH{85o@#u=}Yf=szKB@5EcU)%2gXt;g>1)Jd_r;OAD?G1sIj8 zX2v=iW~Ah&0>`EoOR9F}QQ`=rcHBCXZ~Ko4RJ9c}H}=OIAhmdW5{_(t%xpzvOA4a_$jz3e>@X+#9t zu{ncRmFSq?`TK=LS@9T&GaCy8%J2>pTc!cu=(E#(kxBz1nAPEJ%EGquVaEH+W{TRh z@}llGOGEfF{i@vAFY0u%Wi|lvcxh1H{ha;$u9<-i+-Qw@H-%SyO_9qjPq2O}DstWA z)pREs-iMWAM()Rjr@Vxjd6pouq#GeH;f@`?oQ@o@j&3cZ%^zXa>`sHHs~xvQ9uoW9 z$>*U4b+VNa8={X@*rj`7j`huKaSF^RDQj08XfCrsZZxLsC)lxUcNphNaeNdYo`a7X zd8mQCds@WOVtGjep~5VcH#b!SUNQ+cihB-?iAuwr&icCa(|Da?)@Gp7lZYEZfVUcY ze~^mNmdT5YW~m1rCRHFzFOH`6_jY6j$)$?d8F4Oh~lpZ!l=SzFoa|+6k2dT!k&p+&uQR-_G zPYz&F1zu$+($|OyJ{es#1Mc(X9$ct?)KSPKApgv&)*hR5 zt)YIg-m7ZE-aihI*-k=!= zkflj6R{O_dN@NEGsBg;dJ{v{Zypi4b`MoKgKLv zpzJBMU+a|2J8_b`7ilX4*Y3>MA6&HWdKl>5ne-!0wBLB}LhYXYb6$6~anTMh(hj9| zQ-^Xkb%S|k`4d&iMh_ZCcTWtl10oJV5X!rBC@&r9X>4SvhE!{0zGEMP`%9kQw0S(P z4ZN<>qlXAd@w)O(P4v8O9Rfd=Byme|&4M+D*?({ZOZ2(^v(Kr5$PnRwME6z+3_~my zytIN6gSgqk$P~`9LO$JQjyKYxGVH1)4Tpii%+@G7%c$4ov0OCCWP%gO^w`RNW7QnxuPBoL5yisB z*A4NT%{6S{Z*$Ur;$OA|_`hm`Zi*|j{>}A^dH4U1`Tu?n^yWYcbzefxX6+_}$DI7Q zAfTarkBcK%MZv|5HkIF1 z5G#SlLO`J1-WCxM=zL=cB_4c(2?`O7{Kf1=MieWcdQl4982CSz^bph!0TKeVb)o5d zSyfdC?4A-0M9vRqX9;QPs=J4WXez|s&Y!YmT#b4flq^5^xg4Iz6tc;AJ>dx&3Lh9cC5n{C!AdH`% zTYE&+$Kog8s$)~VcepR^=ooQ#fA8jx7ud6rts^cgo3PU>01f^5QOSLv&g}LEaj3XiNhy|6EyoZ_{*oV8LKO$uwTgS$nVu? zr=M%N)w21^I~!CfG3)X}QbM9`&F$4XJ{F#?=oL=PN@@(>oR9ayu*3p;_o3|@3#JdA#qjJ6eu@9h~4?d#xEBL zSL@?O8ws7QKeGV}CnIC7KZokvI-Yduo8P|sB%#t#5o9zpGV`+)eH7ID#c|mtZB+#s zJllL^NMO&qe2eoBM!dxpZJd&1$`#V;h9M!*(YxbWqmx7m4cJ+EK2r~Eby~XJC6}F@ z#?QGXS9=|v8XBEdE$_vY1mpS++--f%fYr;J>kKca^KI5qhaM`rUQ4D`R$t&v$ee30 znbcUS9PcM2Mvn!ld9m7dJ?lj|wN`J2j?;h&q81t*&JJ&+A>7ziA^6xdCX?TL@rY(= zFNsoS9M%<&uUj2WD1jujv^unDoT^aJr2I^au)T=K+f|nvr;37#^-?+CrA@{&8PrHu z59TkoG1rfSF_JWnwsJ13?h@@uX)uowyGrm$TL_ z)7YJoxU{&uBgMn%QWz2<@Fz0V4LB@b!cz;nxTH+{U4v2~8cU7A^D~lz%mef__$U?e z_wOR0CwT9BC6we${Ia$&N{PxA-1kw^Gjl7X18S`?QBle;awIQ8-+tRT)HC91k(p_h zvT!)H2+Sxd7IqW}Mk~)ta9xd!NX;_8d%atXIOqQoVVu6SNNs3zc9L+Hh~G0Z7Pqvd zTJcdbz_+S@H#g%27FQyESqN=X>ZJ~IB_15gzZm03oB`2qFRD_uXLmSjc_6`938)oDf5`v2U^ z*gQajlZwf=6yS^W#0fM=*8B#3T&9u1$Fto*4~w=iH#htyq4#RN%*Mb+nO2aOC$QXf z6AfB$g*eg5nh4NyVUjY6Pl(^vl_xvhbW=S!Jx$8o+n0&mALSuzdpJufTGXauXJ4-K ze&-t5Yx6$sEc^1)}+vAHV-Gp8>s*EOe-uZ_NQ+c{&Kp{3x!%o11`I6ZM5yd zzrX_%c#x_baB{X8QLX&?>;^QG2C~>`dJB#z#hAI5tWMH0BcY^>{%x!4tz0Zko6MQ# z7Y@+MX?K-twep4==w10H$Rr0|HEWgVGO1mk1cA-&F0@eTB?{_-82o&D;BRG=zs6k$ zv|mzEB57<~)LySo1fkQx^)$s@cN9Ka1l`G{NGh95_c@+oJQTgI;P%#HAUGq|^BQa{ zNdk6hd%kiXPIL|9E4AUch$Df!q17R>i>B*{(tt7Gu5G_pYPHsY5!$N-cU+_EUbQpf z;&#dHBJmYL_ho-MU3nZF6uAc_;4g6AtLC)nYnzO7PJK9EZI+eZ5;I!ETXmZ3c(h!7 z=3MG9WxZM>pb!_=U`2G_eA>IL!!>fgSW{=T9Fz(J(&up43$4^Xybl5iC$^dC_iQ~L zM3Gpqu(0Hcv^XIqV9;5)lG){35(5(-r1|%WOb;Rx=g+As+^(mX&F21fk4cMv2^uaU zcHLNXPtxbjqt0j;kp!v({@dYrn~+IQ!`C;J?}711=@VS8pB?c~{2idZYa}USN#?V~ zBR-##NccFf_;NY~cq{knu&!Hi~Q|EJF@t9soNX6wgG(Mkzh<&I+Q*9S6G%*<{#Vc>YNQf7c^ zXgTA8UD+NV%@X*un)GlcRuRTMo@N%YWY{2-xV>kY?DhN0ra1|hpF@*-&_3Qt4;kkK z`gyTZc)>thNdZYOf&CyX1o^|7Pz_vkop+2EeZ~yTXw9B?Cf3jlJGZ^Pgl;>-Iiu5t zWP2ZNm+`Im1h??AQKIM&`)|*;MJ+9AogYW6SK5V7ijJ{x3c}KY!UOW`uBBS5zd2xi z>A3(L7_qD?0#S`$qqA0$DH0@&_D7QHKm#u_V&26%?~gu#vmN=WIqq<@-H(eD{BLsJ zo@XUSP-$C~r0-AoJQ?}Jzk+s2fG~qlMs4NtU!Hb(6EtnKVe4Gs8cf({fW}cFR|OJp zzoJTp>AasU)3}O+Sg_tmy%F$jr>Et^T?_;We=l;7&|3ivO8^)J6FHU2B^p@|uLeq_R+4NFX zO(Z)wtXQ7MY4(v)UHCjT{Uq{vUi6{rdbzh%>h47N@==W$wg6QFj)azEe*}3R{=6LV z0|uX03L|BS^Ajs8FN;fJrB>AS{c>;*Z@#Uo?oZo#Lbut^|5h*~mk5WR4pxaxmib|^ z(P0|%cCPx#@BLJOM-`U_=fLFl@@>9y3c;NgWcdXwRmqR*>KCxJTJwjtcu1=v_(*P( zzu$p|6>!*4jn8eQ=dR{vvh}JS;2-xrb+nYuocO~za99~73v9E|IUkur5r{2tR zEZDiZ;PxhqB+p~p&hqEoTzR!xJwGws`g9a|;Jz-r&pqD#)6bhy?117O1+EgX!wR3z zP_B=sm_}--jVv=4XXj{YaM(Fq&)d~cQXd%X1%4tyefQz`0f)0Iz++#N08MgMR@PCM zu1DbKSD&}^GvMWP1xEE`e`M|lyqHnjDYSIds(|9-nvaJHfpK941%HdArEIBKM4nTA zJh7=2UpQ8rRBp@U^yrS~h1I$@BGp~)gHlp|bkFC@j~wi!F5P!d35jrhLY0z(AC-qo zKMik=y1W&=Kkhyyu%*_QL@X=|yQIDN#kR!QI`8(o5{5qBJzN(^Qg&TXqT4-nzolWM zm2120oqkgDK2Lu77Z(cJ0HCVl``t&4OOT55wb%>T%jZ>Ub!thEh0dnRgu?jKB5>6`B zyf(C?q?A4vvk3cH7R({JP;dy{?) z*7$X6)0@Y?0NNR63%oP4MdA6`0!nwEoSf`$`aCJ7jtdRf({)r+o!>2!)20*K)_9<; zLN5x8%HdNChyCh99L?|UdhWoXFOitOw^pj~TLF%bdDL^bf6w6xvex+-4`&Ufdip3( z`nf3D1wAfY_;x6p6AOd&T_9985o5MkjB0GgOhw80DAx1!l{ao~55#SIeo@XP%fgMA zVRrHI(Fke8$8cRF6%r>%S0JPP`C(Nd#yXbOYK1U~EV_c|yKk_gXBctsUg$fiJrxDz z+`RkcFR@jrA_U8jlddSt7H)VRb)#CK3Z>fg!D!A8yK83|Kz|wK0#`)ryz>sJz z+3vQhuusKv0L_G{6C{*PbJ>}`Wh%!X0W$$E*S~q+>)&o!JAtbdshPFJyMKy=NsDNt z&tcOj!YCnyPASTMDtL)%=tU3d!fjFsn&|kY7kO<*_|QypJ^FVR(8gwKJVnb<3FUmV za0}fhFf;qu|B1Kd70UH_gC)^kx~>08IU18MMWZkb!HTfOS#pfy!z3I8T8DcXvV@9) zY9WxGj!@A$b+h*J6XhnuR5zCHZC|%SWIx@qCPCS@onrH4!tmzIak9^pLUhy~-Hm1W ze_9>7&EU}&Kt>BtqxD=lzp?j3Q|;bqPx)HDkj?cByKqAQjz{r0pVBz&j-asB=V4Zi zW9_9K>iRB^lKkrB#VF?Vc^L93^(XU!Y(Y{0Zvc6AvJ=4{-hz5vQ&)5%qqM+9KZ=#| zOFd8$hi^Y_`M4s(y#Y~c0;k$TmPydhM@ACn^kQ=n4CraJNaxx&Hs+HHHe}>LK)?m? zGmGzdmH2V*!5C;>T|3KcHDAt(ujgFu7NZi0h%VRBl;aIqUyO3(7!TJ}&;%<~Y?O~X z!uC1S>d^ixJf5WidASZxvxjl2(9C_9IF^9z9GLURjYyh&+>$n=%|yTd9swO^gf!k6 z-T3OvOuV0bO0mArFaEn3h1t2n2p;Sdn)bPRFo8}ZJ;BfLr@3$ERM)$BIFlLEVCkAa zX`4CK*+6W;(u1eRqMokaZ!-Q5S=StvS-W>9Otx)Iu8EUv+itRLyUCtx+s0IrYqD)T z+4!D$^_}yc{(P=$?|t*fwN?}BadBH^?o0Kq3}dkv<=85Uf&pF4bO7r)W_HCb#%@rY zuu%+p<$87tFEieODM0VRm(aiu@}B)N5D<+z;{9ZFhlK2`4=(_3O;>`Mdw4^BC2r22 z37hdB>a59pQR`;}XtxU(`Lx)1iNb#d%yfyjCzo`uzUk(=L~Ny(SN-Da!i=XyN#@wi zX7;sO;7a7CO%h_i6=Q?1w5T8fYMvF~wWkM$W-ieG>SYct#0m zn--F!e+fW76Qo+@RzlG;JdavicB;7kLbrw0DO&4rk36RkG0VPOr0>(<{p`|uJR%~9%J!oy8?jN~g>K3;U|)asYbRtk zmJKx{GA~9-VC=)3RhfujniXYuEbOVNr*&b4dM#Yn=j$z~Zq%+XVu?_8q4ZCkQ^jGs zfdEDrt^ih0+jz8M^y$z12>y8(qgfa5S2fZQPq#HXFP%uE^bcjTfP_+lOpI$xe1|0ga_!=Og0Zy!WHFVd$b?LU?(cw?4O|jf0C>TD1Tz;Fv^2q zs>>2?)l$JNRw8uE^|bShn>0s3(}3%RWfw3n9}ZkO2Rh1yZ+e&J;GN^>z{!4 zUczGpWf76!tIq6~6tAmX$hqjZjqc|~;H`Lwc?U{i2C>CR24?qoT7 ztq3Q5ix9*=!@YG!2@PDA4l?bful&SR7~C00SF4e}A%SejlJEGd30SB6*~eu<_=L|e zhtfjugWcW-T+4%pH+YYI2Yi9|7^FjpgmbE>VAkAH#1JnrCUxrI+X4q}1fVUZ2QI7$ z`TEs!dAjk0=?Q16oruwALr%>VP)L>ip)^kHR2ErR*H_c~b{s?9pr{RmmaXqshxw8v zc;UvG6{F=3WYg0+^Bp5OI>TUbmdWqWb7BRZPhn`%nIPR+Cun>F2uDv-1nDE=nXLHx z7F<(zP-)DTMTRa|00t6rPb2`=O=;eC*c}%N*(KM;TT>Gqm1+FTFu(^z_9Cv%)3(&> zQjWK};3}>fY-pWA!(mPF@t2;D!bxdK*#LqW>RfXX%F2zRBQpWQsxr~(f8KZh>!_Ys z@SwAtsSJdKf<`<3tk2Fcctr=J3vb=sN%@Pc5zDcqVfofP_#y6Z{la*n;>9|gvXiO2 zLav?L26o($+nc>YDDWW8{e$1yzE7dbaZ~N>h3XR|l(sXTt+v9-9?w~%3nx&-gUtIX zO5eujB7KL+2DcgK`snO^Uf)f#(6`lwNVCjbJ3zf_nL1%Lbj&IJT^?cd@Ry3OT&yZH z-47+G{WNm_pHLO2fLJZt*Amr>`IT~KkqxeE@F3G1INk!6KFjY1Y66Z7Hm%R+A@GZw zZfP$A+3#sK2~wt3Re82+O)5=&60ay$?;{Nr-$T!iM!1(n4lZ;Pm)6fSi6&LNAkV~J)r{(@+_ao#9sZ3(%tUIx4kLf-KH3C z>78{!{8bZbiZ-=ysBm!m3|{LpHR;JWcmG`^)3hAARc$|i9|ji6 z=ZS@`vhOfx($qcCUp2}ENRJGb8IhWT4#6L9iU|g7OYrv1%^8={;VOse*J^T%1u2fN z5KPB{RHKWOpCxOZuULIDB4!g$RUWpj)|#|<- zL3MdkQBhg!Jal&+DcWI~Y|P2Oq~Nff2@0cGV79OOGTieuqfo$+?1^f&XV~4rzLure z>PW*XQ;HB(rxEgdnm=meQs{wNWlAeEg=Kch))t5tw78f+7KIiWs#8Jn)cBkAN|%h! zW4WoHUdt1V12I4ULA#a}A+oxRFAO}HF6O;D|73h5D~U9J1SJi^gAfY$B}D_EJHMOEHn2@EiAA@%ARi<@npsYgE8AO1&f%N(Ny zYs-3@5*yHTU}rzxC94(@)9Y(xe$*ptE&Qs8l--oxxn+Rz`jK!G>sH1FrGA+sSyY^+ z#)H><^h37zQW9wziklRBr+vGfj5r4)v`Ck}1_Fg>9upc~3UghEG>ldPMQHC5Rg!U0 zH69V-O|)&=_UXekT#P2^x%Qw(TC_E>%M>=1eePf=pc4d~84L$Ge&X07<23s2!d1-b zXmjHOA32JOje5`8^r7XCnUf_on`!fl766ZofjGe@T9-%ht9EKMz`|5UbU`3^s3scBX*%LU#n1*Noz}`hWAOc(ORz;bU$iDH z%)%+*-pA>Yzv>(kvg(r(hjGruL6CqiC#Ce~==6`tLz~C?1ad96+o4ZR zpfuS^%sB-{L`>OC4=1iaC|cGu5q^~cq0v{oFD6m5Y$%9H4VT!fK6mUKHU)J$-e_Jr zP;;^=(dGe{E|V`NEiIdOEqBi|Z62>1Z!XU}?q?%YE}P!5M6DhnBSYM}?VW`FmV~25 zx)A%$XAbIZZa(dtZsSO0^uDdV9GA^w8x^BNgvJ-PL1(*wNcVr0y?`ZFQ(^vzDwt6d zV3YsvBE{e83ZZ_*B#bQo--rIkJOJewut2^b5^16T`w`+zVtw!*KG6U3yWKhxBdEiA zPE*AHz7Hzw72-RvOV%sgdNcINn=iyAs{yKQX#ukHBb8*rig;1h$9(ckd74M~-k z4WOYA?@1pwH4s4XmiIG#Kd2h^{#U~E=Nv@*LKQe2~CuYC_S#2?__wFJ87APNXXy( z@?TT$83_GQ0%;D)(Qnpw2=$Zb0ILH_W@(2Leop!YSLV<&n^i1#`O0VEOS+F1&VO29 zIR8ixe~>gcr@Y#zXg_VZmP`Xes?B@E_>SJl8A)O>Xjt6m@xj?Dgt>SxI1x5=_7x7z zJSWW^y8rI%0^#9C;OlCj+d;-n?%L43NT)}}Giz(4TQ#mqDps)@H_Mjg}E;#{RXe97?~x zg3`mLjf}k2Cb%)fTqiV=#`e}x9QrbW-&)XKiNo`m>(49PWGT3xL(`;UyGsXI3A2f4 zM`xj-NUSXX7Bulk{P5?vY|N|@$s;jGgF3i@MBwB(%Xx9QrYjQ;Esz(m9!ldR7VBe0 z)C!x@>gp5+Gt$LkrRR^Lg{p3A}Bfu65m!4y%(dy8u3#lKV0%9!;%m6JgSQD=wrr;^;{bs(vP+aQ! zgdP+~Boq`Fe?C46)KD)rlvbkS1q`ywZ2c*j@gJcqAq%LHB>$(=HY8kBx=fu+L0@o~ zA8|-sklD;j>hyj29n%)5PPe|&$9LjTqwUY@&$|;`0Oj0`IMh(EZfvo4I1AGG3JtpQ zg{q`4^jM~i$Zfvxi=2ROOU{eg16jWb;$xIbyf{at(5Fg> zVbB#(T5(~sJT^fkq4_-#1SCQwV78?@2P*?91kNgtF1GvrvXm`pXkVK$ z&imi77@R*M2&j<|Khev*VM@8n13|3z)J<_|txqhH{GyFtDT++;Pc#otPbX(yr10Y* zKct4n=2AJ42fv(ZD>CqwS=A}Hh;;v2I5G?}EHdy(?rYqojz=Djwc^|+p9iA`DF=tT zRo9Kzi`THgrNm?XiChm7Zlozal=P4gQ3*-e01Z_jcbG)MR!a<*7kMJmpT-D8^SM3ILbm-g5p6-HTES%q|^6n=Z^yswEcG1rJ zbqx30#szX_QlXhF^OzD9c`=)xJBYkDIb;-M)R)*S*Dwl1uK5;0jxWGGmAJNcf|ysh zk2(6P0fh`~L8n<6L$4itWiS4dpOxRoi_bU(q04dl)>GKPrV0!LE)3H`g@ar{fOLA~v%=9fR zEn*S4B!I_7T(IQE6m_U=bQH0dsZ*hE>qh2Efc7wy$!Jg-8WO>3NF5?sY+FeCQLJwo zQp}0R)gDX^qOoGzQAe;!j{dCi|$XcG?_%0xT$g}6~r$ ze{Xe2&l0KK1bC3j%x*-0vz#1zm-_>O?1#{I$Z+VLN11oH^XP#dPn5V?n9j};*Tmp& zcO9|v1GNlf3~AU$S1;F-qvh3IBr0=_KtK`P8D;jGT|a!C*W=Yp9SyIWZ2afr!7No6 z(ifgHHEXfYEb?T25ma{Hew-VYQb`a{!5Tf)_H!C1(Ehhm0LBpHphklHvhyMW3(ViP zn>twJB7Q`xz2twG;3syc9S+4PL;KmzNtjK>=|m)sJ(dikZgV_H!`g@BqE2W~vWab>{!;E7AO!D2TEyWO`dl z^z$n8x^?gR*@I*c!zIFz{4WVJ>V|}JHa<2YOi$BMzc^YNzyKlZlKcUj6626FQ(rQ< zl;ij_+=;$@c=RKbF*a_n;B;$(@s{q`97?u6x`ZMMhk}m%AGjr$=x-PJEQCs9(+Cv9 z(KWX*k9%lQmyndi(awGMd*-H?tB>J+<))y}#5q1O=?`&Ls45PI5}_}7$uU=6%Kbb% zIv#QdnjX$fMoh+D+tb)us=;RCg}JW=Le6~}OEV%TfHlckUS4j`XvjGsbA+!3@O%A( zNBI9o!OnmfsmX2lmwuXa4W#z4t)+P%HK*#z<$<&p1ZtFBuZr*aPho@1zH2wwSyADf z7?vSwF6HQJ_MFVH575|)9+-X#A7C9wZc4KcoU0^Iil3{I2!PIrmXO& zt7GZS(m_Eo*aa(58b|_Uno>g+3|hWE;$L(QkB`k>t3D9?k0&pQVL7= z@}kR(Upo{c9MX8l(f-Y4&wH_Ji85b6*9vZWlM`9(x?y_YQ<UJI^Wb`2i?Z3G6{L2aH3wmc=C ziQxk=>HE6=lIG2 z$C+wE;P1tw=LK&OqB#g`YEm_~u);$216B_jKY3(9;_*M6AtNdIM}<|8|IwMy=O)RK z?&nG<$?ejOk590}P@L^~VI<(P@&*z!e%Ft%2taIQbB>Z6g11mlW`OTeigU6vIoUm) zAEu%Er%pkS0PfY#H+N?9mGSIh;PR<5<&W6}3HFRbB&#Q(o8e+7$n(Z3T@u@7|Z|8h_IP*DW_Oyooj&%f{$0*;I0 z*Uu7BZhqSQy+KT%Ho8!AA_R7$-CLjs(Q&`vQZ};w5*qY+K*I9bDmvnPh?eDlOu{tg!_fO; zDwc$#G=~XD#BP3bvszw${w1w>7|c^4q&dA-3;)-#R0^FgcatzUjNj4|z;&_nCt{=k z4?4E5l1E3w_*u+%Tch202vb^FzA#)^*r-AQR4jSDU#A1=*%h=FssOXQ__zy3xmQ{A zcG#`Od8Po1N03IY>?59?jPMk>{=T+H?x>Bi9nbxd2HQWgRR8P9510j-S}SA>4061W zQ$nAukJ0eozvw1-*>!XsoO78RIY$D)#sUYh6VP509o#lS>DMBua`b7;YB)L2TW*9_ z(F2|+I1r}id;1^XSM#9%?saSm`S+AefMAyv;ys)DpkrfGnVVBN+$Kd9yj&QRR&eJg zh&U-Hl2H~Pg7-UV50{lzqEw^riXp$>>L73Bi`HDt8;E$S2fox$%_>vpDL=sJvn*wgF!=uCFrJOGX4^OAtJfh+>7$~tMuOdH+O`Cn;?*6{Gm}e-kJ0P>Z z(x&tD^bABqRVJ=?>6V-snfxk{sB_)ms+_)P+pDj;U0WZWAFEbM5jW7^ z8wS&Uq0)#Z;Gwpy)CEJCsYpl>6Xdmvss#DtK508a#p3<&$2wi{l}AXzkM9FYDy60( zX2;M+0fF9*`*7yw=5ceD3!|qRGkePe3x`-6Fm#WDbXxq5z_%sTB#Ga!~Hm8NCU`SsU{F%YxW7Tf53K<|FA1}`bU8n7U8Pg2qU z#~8ezkg-rvY46GjX{1@c8@hRp;|z~X#N(>16LCq7Dr;>i@A6DZdz~h084uo=D=I1F zqt0IKu>A!iEoK`2*^`k2) zD}N&F-fZFoJL)vAuCGl-b`5Eb zolgugyb+3J(@3zfj~r?`1d|z_q4q;h5_d0x?$Cj_Fhiw{sgk1ZgZ6jT>_5#HzZv*o zlwZxm8whg+K9}n#Muk3|I-{J8POb9tCL{_(C#Qq}kbw^n*tzU{chi15M0}U;0=9Sy zFg4@jw@IjI*7D8E1f+c~@(D+ad2ibl5uG_lU-ymP>9b!uHxfvv9Hi4Z6eL}3Q)uYB z;oLoU0fdi~kNtcPV2^v%zWH~9^|f`XkzOp&^(xg<4C{t}SWCBVzouB9TtZSlGc3za})&6I9{)K6;bOcnfIe_ z@IF%SK}lzA+ofAUzPGrr(DMh_z3lb2WuDGow4Dq{aB(DYmSqErb|CDx)v4D@gTs@$ zVPQ44K*QyHJ+PKbP%~X?*WVKmH|ewQ+MuG~FFxlfyt$9;Q0ZvzDx(vymcbQi>> zVZZHuIgqj!y}}29{$O|uvIR3cJ527tHCCXhe*olB**LRwqwwN6QK#6LNyh5AHui}i zj!fapLW~qBwKeaAiwij?q-)xzRQ^W0ocXpEzs#>&J3#Gglu}XsuJP^v$t&wYdia1u zq_C;oW9{Y&SSpV`WUo}G$0h=YL`?1#crEd|8QeP_QqWSOS#dr6PzzP!!UcB?k}dp|G1LZy=LSyx$gbxWV8-8h=PU&R*NG<3KI`MY1ZokOAGc7 zm)&B9oI@z?<&}|m&POyPr##&E5mwo;*7IJMmLph-ZP3bsKFfDYoUVN8S|Bfv1byYz#H7vI7A`QYWmUcfm zYbK}U(t6vx&Q6fye()nBr|dN?(A5-h8hhUnda$gg0LC-GYP4T7XJNWEJ<50{k5sfO zK>|-*7k+Zt*WE2S?oY1XFD|V0+uA6@tsmM=M8rf++;p{L10RhL*%;I45qrJv zfwsKh8f;hs4xGilxM0}F0WQ|N!@*1jiYkNEfERk%0qxm7WQh1kkg79%*#YuAn9ws7b+TA1xoh1g zOPN{c3DC9LTM2P-873%s$aa7Gs3VX+7))bZR)62T@LS*DcOzgB90cZ0+lj;gi$0c2 zSn&Ha6k*R`k?5M?f^~{+By%pc{%4VdG7pZjYF!{UOZk9fV`Y8asebvk$bi9( zxA>baood%1Kma8%F01JA%humK4mV$IgFR+;Ze-Gwm8TBDXCAu z`!kahReUpIm$C88o!d+WfF68srVtDAmEAO`lb&o_&9IZyG?QsT#mH{8-*{imo}4q; z_$zGyfPtEd^G~cn{#bxSxSMr}MSBS3%bIM#nS-r_1Ot3=XN0V7&*gNzd?_mPZt+Z( zfscUgb#8Zi{;SXMppU${je>{_M^L;5il)WRL1{Mkp_67v{n4%tUFiGAR? z|7)@NZ~GnDlZt}mGQfxY(2juz=q5boRCa%BsCtpNKTq@i%e{HIAz@i`+zlZ*pLy6* zOzIH)^T7)u#;+?EK5lKsbblos|KP=dQ6vqhC_it7?cnmidwEep;G!l0B7Cr~fu0;L z!*e~=398MDB_rqXh=5{)ZbTyecLFOftm>VeOadDWT7r!&x*2pBS z%r*{?THQaUW~V^vrvcKA4RLW`O~8tR1!Q+rnFWi17Zg7EHK^I&m&HZYAN;jBm|&o* z=WJ|WWZPEOT-s(ITO|6b-|fvVH0;vlkAwyuN9`8QSM=0MI&!zQ*X1{QkK8A_XYhjyupt{p(7`g z%(IO&p&M}L>+QHt%q`!>Fps?pb?*PEUx^bg;Uw7uyO zXC6*2`CKp0SN2Q^bgyHuTI#epg@dCUFRKvN+c!s`1`{tylSWxyU?)cF0QV<_EoP{aIS#r6`gkO^Yyts0AcpWVpv&#@A-(G{rS3H zG@nh0o3GD%W4Y&{?H-)GqLVAJ9u{J;^P|uKcCz0^PI1?{s06Jp#pTf>M_dzXFao5- zp(O46pSd$o#ARw>hK4oNKH$s5F&3anEkiW0`26ygjiG8?`w-wryTX@j@-0+vx^>&W z>XE?;P9ldyfa{o1_k4&|Oojx76HbP|$FQ%Dw7^6!nD#XvlFasr#N5_;DA$*W2yCy_ zFc9*qFL$*L>dS^zG+$;rVHqv2DU{FoK-Sw)^#YHxyAaQHvHhfu_C>SjdDoibI&+Ip zo8`9~Z}4Krvn%rHQTe_0hBwo5PUY6~wC+=R`?rG*0&AQiXX|a&emnQT?2jAmFE1{8 zZ7eTe?Jn2XO0((DN^6VPgQU-1zT<-F| z+(F?L@2h)2wa%BfjyFr+FTD$Ib%H$CDdPukp1SQGAx|zQ(fjTtSD_=5ZfZqRJMa6< z5Wb#)fHh{Bu$Zk8Bb;|f-^VvMUyTkItX8-Cvuzz*ZS39+dYZ@nQC%&9a6Y>fUBtel_X?sy^;>o?`#}|03GjhP7WD6 zpm5cjWf5$bt2;K}*#2m>b9Cp|)e6t`&_}&?uY3vIjFkJgm)xqf!6e{dEqNXNFW(O6 z-=w>Ba9X}AJ3NY%Cvd!Q!u>H-yY(&Q`tBkMsN5WOR?9ypQSeR_mnYOPpMNjwu>Sliu=o zJ1SctUwh^aT`a#8Q{0vuC}aR09TsF|Dr@VD_ppQ=^xL|p0sQ!QjK~BFAGd=>VP=@)7*%ofus{G?5TP5FyS(RAX z3J_ilu@M>pn>0Abyvm$1=$$R0nKJZ#HVNqdiu2z2&Z+#CoAo@Cb=24`{qAh{r5|_< z@#!<))85k`0|Bol2=`tsUT>yCI_(~-j`uv;nZ52IZHOv^;(O28?G8fTF^TlE*VBW+ zWegnjEwt~Q?JgUndwbWL4ZZidzra&)y#OP28@-c(qj0hZ-S>Lg=3j)0A>n(qH>cy) z<%ea%asB<^{KC=-A&VEXIHt??s6n9TJorq2k(XnOJKqR*=zfO6e67v1dWx%PCU*W0 z(_f*?5?rF=^@hN}l!yfs?~0VPzc4pfNs-xb4q8V;%y)xom(viD)9pLrsmd|yswT|VH-{*d;5V32a4wCofQK!SQ zcoeX`_8PnQxC~%mZ{fa=qDiXq)r35MUK0|104DmcZrN9Fz_%MhoVZM0dP`=u5K1!( zdk30)0hk`6*J_!*5KMyQ;Lab6_J$?FHo9BdQ$K>UrgA=xJ2Rk$L zHK3?8F-w5lPD>en97AV|&5UV~NuN#0x|H5oM$^vX74taA>SwdOe9-^i@M&;sU60|~ z-0p~Fi$CK%U7^b}Iq+R*yLeJP%To?);LGVEP(i~9&Re`#(&p5=!8wt`yFeIC7r0|$x9#t_z{|$&7KKX^De`x2%)c20 z6~@O0X^60@az;YeITfd`k#P^gEgo~4^6)9n1}Y^HT>|I0y{`av)x}-`X{&y9g$*z=-{tigGHz z8<<*X*`ZZ>m*HRtC4_M%5;IbxoAVGK_H~EiQg1o@oe`YECYkf6Y!gSJYf<-Y8#4{e zn4iKqWV7ICvs-#11UQw?dLU<+Zl&<-9^RL@5PPVrXaI!HNaASF;Noc0o{%$sU?LP3o77XFONqt+vTr@VHzuTGS$B0G;;;f+%HB{xBQwuKV!Z#?V2Y z9gfzQ>)%@)&pS=O%JLlqAXXu!<>}>bum-3RxnT#S(7VA;f;EfZ=XJ1kU_#F;UT`*q zStAq~L_-&6n7-)GZYH!kD|TaO+Hz|T+ALxfpBndgwPId=dsZ%FqGl0QO_$`D(SYYp zQu&TQXP^FF&a?ufXKnak!pl;%m6bGmD0P}9iMokEhxV3xi#M|vjAjHTVUc09rf0Fi$FJ-S=OD=pc|P}`K$FA-n$f^F0ZrF^AGNda?$S#Z?Hb2R%%N3 zB!Tn>GTF%-j49pq`ZUh7wy%G@0`1jmrW{T_lS1p~)dGLK4L?lXDMO=eRtYstd{thd{&nyZ&1_1s^U(&pK=71 zj_NYe`b(@8%^evh*vV4vI&W1iyTwF(6NLukR)A%6##C z@`$%Yh;$ZN3!|W&&N%DY$=O7&bJH1rUS}Qoa2=~{BJ~7wj04k4srQK$zfEVOrS}Lm z`GqZCqH%SpZP{i%R@8}x>EoBa;!RIh6CF9D7uNQ8t44iOiwh*e-lh8Y;9)(c5?QyB zX0>_Bl~Rx3185@OLL5iINsp81(usW=@g_rPoKFH`jZMUvC)UaU&XYw7B<=_&L{;-K z^tNnkRtm@MY{T2m60J=5GX>kFHe1R9yHn11GuS5brL72I+=WcX&57!ZgIH4-;$S(n z#sO|$a_A~>G1ZZJfZh#FU(ZT7$YIaAWW7KB$yt$)vnqZ5IBXs&_UH7B*>-b@;?t+C z#`$Kc&ccZ29MdxE#V=Zg2tF7k*jo0V*lyBPEWSX&?+>}{a0$1{6Kzfo#7{Uf`%^>FqszI5(OF=>MMiN4#k!sFiI96loMM~kKgTWI-yc?$U!M`m9cep8wyc(z$Fj1F zvsaRu+)C0eoAEFa{#e0OjzPB38O7`poJc$xTi$ZS$aMe=B9@|j(GB_h5`{inGu@&+ z5>*?UgIaV-Pw+}6K~~dKM~#f!{}SS0VJTMEdEUgij)$QyX0L-d!K`I{r_H9MYS>)O zwlJO0Q7$g2Jh&>8ML0^%PB%U$Y7SPNF|FV5^9;o7vlnc1SxjOmOeKR1Ttj=%AzUT$ zek%RfG!pAlK@6_FwNGrC0e7_?p zEY5<^7|>$SbXFuCQ&Y(XX}2Dy2dIh+GC-J%L2n;p6C>kZEog-(1~9>6Ah=&`E3W5j z8`)&X8YCGtEQjujE2`1@ZSdpIYDK7;w60&wbUWcsbSzCc31iyHpg%stq88$jLt)mb zvh~Sehju$q2?Z8>XfQ8ywoU}itNqf;-3%*7c;qx9icnHJ$CYpzHb^!1NCUaBOipT1 zcys8J{1ImP0~Hq7wPoeXvazR^NrZE9fNF`S?^!J8ced?NzVJUmVNDf0csIo&j-czIBDY9$k%}ej15>r-H6;p+W(m=0!@@FI# zRyhzfl@C}P`m2mrl)jE%;lWDJlE4XLg#pJ;gaC$F;Kv_@%z`LE)a5N*`7M^QeTP4u zhl71Cf}Gp1Hk)|=+CNfuS;ffWRNuVJnr{AU2+1x~qZE7L;*25@=cr&z53h-S;}NgjtNmRmqGM@~_J_)qLpje*^5^KY~ zq^7>?R2MG2p-}PAp{4J`1aZU1Bh(Ene($5?2=RA(w=68_IlNSzJ6gB%y~s1Rke*iw zEXWmg>t~^gPKipaD>RDfOtvUD$qXt-j5R%JGG}EoF{{LnQ@r|>Y>Z2r@r_W~C}{}g zELTnG)GtvEZ4I}eLptUDh*RUsdkgmf8=w5-YVR2E{&8qKee@Q^yoOHl=oro|Jsa^)cchC9K)evXV& zx!iXcA3zgg!exl>T&|k{xSMHm`an<&=+;1&@%aw0JT%5^@q7Hg&#v7T zmY*$4y8#bLP>G`@^c>6qapIAyqmuCvnJ%?_iTO0<62K9@OgE_6Nuvcn#Py+2Fwb zOpzn%yuGayh>rSJYJxvL#7HtS%x?(QwY|IRM!=4u?y~Zm^9ztjmsSr>jK}+R^9-j= z=~oIsF!#nL>438Ky^1moa0>d}@DW6jRMskAmrZ9#&bimj`-SRXeg#R6>^YzSlkB?&DLvI(Bo1zm`xh9!gg=tt{Y{Y=V zV>EfJ%g5Plp84+)_A5HXv0iY}>Q@kl=ScmeS zKz>a;8@c*;O+I}OGacBK_gy7#c#8B>a)=XWXf7z&G%uBeoRmN-i&Vp{T|nh(vQ=8; zai!iJ6%7*4kkVvhP_s*1D0u#C=Vaj$#&ngj`!D}ty?-p^W*^fB85@^CEVb~(`&hp$ zVq*80)L}M_Qb*%bN<#nC^>f1ymgL}snAgK+C4op5B@I=<*ap2M41O=Mc+h#UCXa(K z8<|~Cgo8QoF4pxyN3^}9OqzU^av_5W2RjFbAFkQRH4t*;?K&M<;E%pd&f`QDIJ}Ae z{QlX}v@R8g0EUV5m13-1S@p>P?JzsM{Wa-Hc3c&@^tb3RDt~@2N#{@cng@Qg(Yki?!(& z!ZJqL^WA^&^6cv#6ePA_5C}H*ofs(EybgN{|Lud%hCZ3YsayFU5;TBlqI7Lw8B`?} zlPEbE<)Q_H_xgu&K%;1{UpM994#|DWmTLg>_@G0?kT30PJ&=MTf}8Q~2dd-uc~${} zMbPgus=Zc_u|0hKTm_491TNQzENy0!Y0Nu;`&V}?q}TqIaG=lsM}q5k5W?pS6$X!! zi5C$j*<@$)C`icb=ITWy%F1NhzS@ZM{Lt39ziOqzbXAhgN(9r@!9)2Z6CUWIpK1=c!JIwO;jwUFI zhV_<#y6f$Vv^XV#-)U1xaG_L0rXJ~%Yj;!c8K(;-@^Wky=uoWIH|bpA(5cXimY$2X z4OYw)K-09lA4^PX8L);sjN+v@oq+V_IgMaWZamk97s+PkQYoO5A8by1CWJy zHCjn`{ZHdS8?{e}>Oe3Pr#9g&wE;!({gL(+kS8Ld6jQHkI2}h5uftF4NL@QO z63BQR-Dp=O7$D-P_$C$9-a|_3wbq@^={=%1@?+=t%Tz=84-TVYW5)leE?63xV<NsSP=2{e9cImTt|Lf((XZ&esu)xI zi1}12ELbCtB0d#n? z-XTV|JBb0dL4ATpkU=21J*g<(@1&A=L}@#whf%bcS4ZRp`&FytOD>sy?yjzi<@&B! z5XR`>JSIV)fFq&Bk4!{N$e5T7OJ^cD8nwB$&jo0;qTndLs_HD_P+ER2ub`?IU|+Kf zohuYtPfuLr|%xa~g5lLS5ca_Gdu&KAGK+|6a{z zT(Az$sn3)4Gg4UZBg||gU;#WK=}lJ_UQrn^uW#iY-}e-;ejfD;`I{z9txZyA7bJnd z9VFL0R+@D`ZKES>U1g!-M5d4`h#6G~iGiiV50C5~ncJ3&o-3a1H>Sc6- zAM=w-g$=NQeMP77Z!53R#C8}WjQ9JdtU-(j`2|eFA<$MqkrfVR3ylVCOsuXh)Xsp) z(#FVbAA(km2s}pq^S)eTA&SP%2X^_SQh7TF8t{_ruKpvac1fAIA%XDSlU?+ql2*34 zPvom&SqS>l_>uvQ{EV(?FAjxMvEtn_>+o1OCNAt{5hM2l+M$*NkcCd#d3Hf^x(e2) z=#q_Qa%CMX0)k$3NbCyhe)d@{~bI1qk=Oss_6#fAFat^S+9I;ihD@ z?I2lQ_!bO(cSd7rJ*KZx8?l9TVvOtV8utG6N}W&ci1z}|K6`~KDBz-&`gIGRRQfEa zZ(di~!Dk@d7#)(MP;s&>k38ll_?F4zo3a{zYCga4AF22~62!zO5L3B`X2PewH(9DH zUazNL2?V^iEe=o%)c;4+J22L@eP6?k-MBfiZ8SWwZCgzm+qP{xX>8k88yhERY$tE- zy}$qSeuKUCUVE)E$DCt~&Tu9@MB-VqYyObY8=fNdt}|84Dz0lEC$wCF#|H&PFbD#@ z86{Ai^BGEzwcr}(^d>14g9_w2ZJ;dU9N-9t4Tf9v9M`F$3Ajlw61IcjHoWE@c0`Sg zFcdhTBL`O^2PGvbViy!%Ci-v9wO!TcCt-VoE0Q%^tczAai^0C2`@)N;;;`w1>O`ka zMs|UE1E?MsFaL2Q>664PbhRCOf*ejAM`$l#{}#I7TTkhaP_lmNv`X3s2w{@Y3q?cq zb&yoxbD_m34NOXyDOVaZfh!uSPn)~YksWZ~Ll?w_Rws93jm^-l9z0}Ff{&3f-y(~A zkqDOE;p*k76;Z8Wr;2xILfcXQ#ST1w?z`rUyOjt;uFI)yd(r#Wv2(59WQ7%kSgGK^ z%cMp%Vw&9_8b(WycLT%bV@Gf5C9hds=@UeO8u&WN2NIzg37oH2v4BI{J+W#5;;Nvm zm33m$jMlbDtS#QLt87MpH>ypm5{hshuVoxZV4ILsC^`f*n3U`C09KVX%ff*P#ekB6)2n)9D zQS`^FmSO9$WIw-T4IHWDb;g6wX&u3yl^y9%d@I-}0L{8x@4{vmYY~A)hg4se`ERMM0Amd*EFai%`48241PpSm3ZQlPb%TVOTVT*=U#ik6`c<$VX)JR z@BS66pSP^OQMEtztUrC2NVq-V&_vtSHrGjbwnbwI)7GOeKJ;7U zlQEQF8q~B*IDIHTWziVl^ytf^5_>YG_I)F-Ri~eS{5wEE?56kL$f-G&t;9`Mxm9-E zR8?$HshIzTlldmtzCmBw6YhOS?sEHYD+Zb0jSaQAX7zqd7b(P*ZeLdW1Wi zUOuhl!^y;IV|*M71{aNO{AmQkk)xq34yzS7YSTYoanlqL(uu@PeBm4$$2mGGLxiO1S zyjMsznQh{LMFptR$Pvil_ueAF@F^3H3wZ~vL`K{3DQ`>bNJOBeQ>BWNd5aq&$g@T{ z;l=fUa?P0~E=na=3EQuEJaJ9w)}gUqGyeslm`t%L9__3$vX-&*BjW*k{Dy%Ph{N|E zN$zV5XeELQYU-mao4qZ7ykm#o!Okc9>)WnnY7at{lvJB9YbEyBMp)?9o3ET{J=~$ z&ZZPh8dpAzpVPQOp&VO>{T;;ZtVUikql+$`w#a;-L|nYr2Jk$|wtkiKithv)pWU@dM!lPEG)Oo4PE;w&FRM!3noP1Q`APi9B^NZAvjJ6)lC+C}3t>$e zx1YcC=qMlRv0wjWDVC|*OfRPqTd1jJ<~SG3pev>lVmrWo`P)9S1p5Hz zi1f)c=q_f-xmU0Jl>^@0D$dm{ymR1`@U{vU?w2Y7%YJOF`%_I>6Ewcn) z|L{WYTBjiuV-rq4Eq_*Syl6l_FSaIPSfNq)bX0l@IhFV{4A>ZnT$enUNnJ}JNXwcn6u`l!_5p+b9-6p1Z8mQ~Tgj@2&NRF}4mIXr zM{HZl_onC+!=hA=#y38V&(;}{u+Duh7cWNEsI1uG<&Y-IA{qz*)sGLiO9>SS#7)!V zzGO#9N!rf@cUklz&`q%FaI&@yc5h0lSJcfi@g#4Y`RzlIRC8L&M17z`j6monkPhbB z=L*U*w{f##30|5F%(pSa!Cz1^H6-h#(;YX8`+}2~4frdF<63w=SBJj3pwS9?SKvA@ zr55mXmKU#>WXd;3Y*$9~PIigbui#QXKr0RXeq!BTo+z+VY7Y)Wx(W7@BUP;EA>&`JDHd*y zDWz?t$rfG;rshZ?Sp7L740O^?pxS4A2APJtYnxtS*sg#S73$M_km;{!4NX z%$?1crZ&h2R3#e;abnxJZR26Gnk4q-Tub!gDO$1rjCMy03dC<~<35s}7H&J}QH>Zf zxHIeKS$IhMQfG58E5^i406ZGl$0% zubs7=3qB+v`~rLZQG50?-o_mJflA}gqlx|8Y)PTxs^)K@n1S*6Z4R<+7?F5YST*Xv zZht|a8RDJhz$D^vlK=#?gDdtSe#%M9Si4!JJ*W9D#g>0j@+G?wktU;ao6gLW_v%_1 zHH6iozWOpVxxQVeA&*2I8tRwMW{)af7~=+Wm5_F|zosDX;v|Ng-vkAY8zP>>fX?4p z6HCrlVo(Nn81ma1sT#~EF>^phk=)sDokZE9tcqfOFmIa`_B@d}C}Id*{viblcxYV# zrFiOuQo}7+`rAM!LS#>WbL=*oOZY`_?}FKJo4%9MC-;K(81esg_dX7p^UCqz#*WpL z>&Gmbea7e4oxDM?gnIf$wylRsw*R z;}O4+E`dJHU$w#Pp#k56_S{RC`M00^I{CIqbW`{NZeGR&Om=nyt-Zzec(It{Kf-#Z zGkBB{d_6}rwybpJo0zA`k8py(>`;7>2j0obSrOq4{{7?JE)4QIj!>grQ*8mb6hs4{=s3<$jYs?|6@)(O*A}=)#=gV;6Sq0Kl>T#a;hge4No>s8ctMKd6bq++Niw z%XMdRr31k1Dy7-f!JWbiQ69GmyCFpg<%cw3o_=lCC=_w)Vs9 zMSf_Wg(}63K(oAj=hkxO^MbYdQq!knj4~5|g?RFWodsPAW?TfG{$&WqQH?_8@i%{4 z4c^X0TN8Y)5@DMCz&D{WRAo6;O&lpx_QeZlw{S3$MeUFQ1E)SSA3hmUS%$F8oH1Vg>>kz!t;{_7Ws z03P`x!BXb~79w3D9CEPGr<4SQs3ch?ltI+yJ7#Y8(O!B~h<9d&^95o>KYGU?^Y1e5UM;2nu}v zsTISRE`5<8titow?H>bnL#8{POE6KG)0OX5#f|I;*REH8tF>AO(rra_z+q3h3yS0a zvmyU!xNduggB_DJQOawbZGJ+BVdnhY{~!0fUl@MbL)VKX8#*TBDS8d<9O=wK@)*HKMYuSgq@rFzgp4F*Pu5&DupCeyk3a?H;=0_=eI=$ zz|_+^p+5}hOmJ`w?&U1~$`ko-@b;z9^Ki{=v>q1nUf1Q@TYb#I_O*^B2=C16!e_SP z*mvf;*l+c&8?y|anMa?iE!^oxokN^^Z(5%oh`qWC;bO7dCO zrh9$yodrE|n3Pr5F-+C*n26c`Td*wzM3B*fqHfUXSs3H1^?IXu%74|Z+Zq#*UK~~6 zAL=tMMeTe=p-9y|h#YaSIna1>1;3_g)4auBRrGs0UJhuUS$iSb&~fRyKB4ZSnMnE$w=Gmr(1&;Vk6Og57Ia%dFVTaxyGmBZk6Hr)5EgpO#Lr&fJ(<* zQs@~)+<_O(n=pKY?Q2^ZRm7JZ=l4q5H<;3awyt-pQwaz_O{VSo(^4PLQXAmWU{Q9m zF#H5}z?k!6X^I$#>Mi!qcRyg>W?asMK^OeDfBw%7dw_%f3nJhz&L}N~8Z23I)`J=p zy#f#=#fQ)fs@5f}=u^p#diD4vOLnicnz)Dkx4gk?01A=Y^YgQ$s5rH>&L z-%?IUlz#v{|2DoN?Ply*5OQ-4aE;)Oi@uxd;_TW4`0R}w=-GGIxpA zb)x(4U%J3V6gYW8IJO+vvIPNmYG!xiY`cv$;oP7jenpP*aOnj<98O+;xkc0o@SD%h<+L#(`#!jT0^#=v7$RhmXNvi94ZuJ;WP--8$uZIsz1v z&%f`5w!$5L*_2}mdG8f113K^l)z$P2H8s!hT(6dxANEFKwWSZ{1h^1tYE%{9Y3_dJ zd!}Y&l;gZLskzggw_V3*C0wSLcp~QfAFtek4^dDf3D4{2!(KFn#lr4{h696SP3i-K z6g-RypPj#DtNGjOWavdodSWP4TLa-8}e5tw$31?Qn-v$>h_j~iF-gd3_XG4=^BDFvJ z0@S=(m6m2{R*e5Fbw3Okx0>_h{TaVOa`H%}uuIAo94tqGldf)6av0Os)2*3y*9!`g zECU#1f_nNjm!nk=tdvaQ{dC*Z>Fn%YXuUuI0R}pn`DK`@>;uHBss*gdMdqjc zcD>06Ntj_i2+?rUy;vtOsZ`qA5cxPl-m4yjo?`y|<^ue*oJwRmcuruzPC}+&2e|w6 zbnL|zc}BQL&3Dhw3<#Ij?XadLZg~mC)3lAgi$r~_P3NiH7aa`^^I{OFdm6>Y>ZpOQ zm&Z-uX}!s;rdSen8^EMMQt!;T)@*GF^A*p@l+r|l(wU)D7-nK(V&SH4%%br=uZT4h zxTvMg%gDvWrBu%rJ3rrWfVzneu#+2!aZz#mdi&b|zpvPq&^t4}#$$$Y$OL4Fm zt$#)sFA$-HR>63~Z1H0GzrDM#{2Aq{=B@BGpyu+X!=uC_?Y4ER{54~6>W6HYX8&CE z6e3oB*&f0qA3L^+Gl^}C+Yb!MO=0N{I)O;h$HEQWitYme8q13+CmA~SjZXgjO6+HG0M;5@tk88~w|_Pa#3DVxW2C2Q$GOMK4EytQQ+ zr%!$qm+Py%2v>k!s2FF=3(gZ-Xr>=y^4g<4mgdD~m-T&?`cO{{WGB-wYb#kdSJuro zO}DRc>ppPAf8o;WwQ26*U&71-@p5`03%KKZT*W#w8zL6sAxwY*d5|#AlF98}Ae+g( zu-asM81TIjW@5BtD)VftG*@wf&Ye%B78(i9U`fw0nw)$>i0p(7y@*lhp< zJYFIN#6#o% zz8q$30I~1PIq&^AEi00|2(3Gp0g=Y$l~+`cd8PMJ39ff%?uKJ7t;kB-ipzlWlIF=` zm5w0XF#KZwy3+obX1i6}+f$7%rhj!EZcWGH4;`0ZH5Z2uZ}aUS=ej}V>&hh6S#KYBw0Dy(pK*xwa_gMWtAN{Ez*?PvBB~~L|0&ZhO-(CCGwd+x;zfnl-oyYrZO3aD75h>n=iP{|4DB`BIKJS zcpVsMc3yK)zxGqt_3p3kymr9lH1Ojk#^e3ypIEH2%!#d4AKy-rL_b4g-u%qVS$pew zj;58(5^3wJaN3^r={O!fvA)dfYc|*ce|hrmY0eBvv>eQGufFx;q{HCoBwS;=hv@Mg zodq4w~ia%M5cJPDmNz$%kgxHP(4WW%?mM`T< zql7KZdwahGwE(za_KMjHG zFrC$t+1PV(S)9jCs5lFP!`a9>_Q8m9q1Fo-O*UzUU42?t%{x{X7v6a7|bx{?vDyrP{qy@Ga3kM5udXoIplu>YTq&OC zWj_Bhsyk*HA>_9J9`jiNo)I78h5(N>;uoF2C;G#))~ z9fAUtwGOjf^xhPDt#S0ZPsPnrbz90-VFf-Z9gS|jGRg8J#NTe`iEk}@{w-GS^w3g@ z9a2Ki?+`EFONDmIn$SN!?XT+#2;bSxcOjMUlgiB5ynpl;HWcb|4;Fn?6CdWg$4OaICqX@D`=>^uK zSEl`l9iji>DJ8_#TAIM${UFOK&(xR#v$O2`n-~HtS^Z9nK%~DL16>`nr)NA58p%m; zNn!WatD`tueNdV~v)7hL5pT{B!O6@`5s0UcY%@1iU^Rq@yMGaLLZlFB7*hC!EGvf3@$ zcVk-W7S|5-!I>@uDRkdJY^QdD{+Ow1Vbv6?{<>?yEm~LAiQV5I|7y*NW3*VzEj48} zYlVdVqR?8>!EVZHILx=rzXl^*@AOVJcd9>2bnOf+Ku}=t%U(wV(OeqYZZ=ChDesJx zcWmjtmPGfLV;j9&Hk^Ihikt8j6Y<_Z>YtWHX?Yh@F^Bf=8m&3z^=iQq{we_RwmaMV zB$uI0i2#_sK6qdCi!K&>BKK9e7UDg`?p1FeJ%OoH2C+uPw0Tg}ufoirKKR++k3VXt zC^6{JwC4+JKZNDBiI1@~CNFOW2Jcet-f5XFt?TG7J=sCN{>Nrb8*K+*pulntGHC{E zB#kBNw!TiF`_{eqzQsl3GfP1JT0daavC@BB@=ajkofV70`T4+q36lvl>|{;Q?QmH^ z?)mUO;Wsa{-mDv8=y9vxxC@V6DT^ny4oylt)X%W~T_RJvGY~?ZWLdM}`N3FqWM0Rb zvi16`eh;egTBv$WmALfY?&EL8%AqKnjaK?8Iug^zaD;2 zE?sgyD6M}QLUaf@$<#GJ@wT@O{RlmX;4mBvGh}x>2bz`nzMm*Fc-d~WZu(p&=4yGO zj-;AR*FQ}xX~s_D4~jHw(6BrGF*uneUTe15u+SYO6N?5I#`0bj-JK<0M2Kf6;Q>}Z zJsxYh~W zjudfOYO6x+@)7&J^u4Wfyl-$ntkEguo6Ph*hJ^d8TfIUK=KFGK!kE4ly5C?eX9Z7G zeqP_z8vcd%@Wlv~5zOdn#xwar5EvLjlQe^)eKU82or^wR54=W7>C;{Yh@TR9c->o3 zB?TKCKApl*>4KN_?==;y)m<+S+Rb$7#OPJ1dwl`iLIg>{(kU-lK}-*uUIIClQBg9v z{C*NaW}8u;qotJ>z89YCR{LqaG^NJa*B0@t82@zw^$_#ehz5;No?iKV#(7677AqAw zmD;0N7Z#hebAW~F0I^`%%$9c4qmq*vOy2X|Ta(Zzn8*}5RXI)C2N50O8D_EarG2no zQhT9@@y{Ievv$jx=3o{w{N&WtYXqb!>el0U7c zbdaOTID%)X-R6_kC+*t1e7C@;JzkD?H%ru5PEo$fNL-ilKcQYVyRc;7-`K4n;d1bN zyu4y{=gkOEzogm8k@a6=RsbywqTK$3+Tw3Rmw}67(rI@0|NNel-Mcm*GueGPG^4px z?XE_XQ)_;#M4NOKn>8o3A&n^Qy6%zarbGGQV~?y>qe-b_+M1w5{O_*Rwb8n6ZyR%#BVUl)FS0H zHKE!%_0>}p{PnD5>i~YD#^*`;T!wSPCJ;A(pyR%+#Fr{71<#>~utOhR5=hHlYYXp}-e18{r1xPTd{ z$J^df3>^z5Lf9`Q*FUHjxc)l;1{4ST*8o)zFzU$IY2v!>QxT7_I|ec$*^ZMGjJcx9 z`bm#j{bv!5{=|m921W*^Ad+RoO`iWHt5di|Gndc1%KC?;I`kEL+=t-k7tU-VP@x23 zyY?ZTc9TamU33SKp-s1xXg!>ZH8h1@$s`}$arU8+O|vQ}wIpphoeRyyu0I{o9S6m& zzpA?90OEdEnzgEl&Jmo@m{U;chpcrpM!3s69bWRGRZLJ&toObgZh2hY!pUMsGU^bi zq!hhgRYU~N$woygMp6zMAZ9EhWtWQ{B8fH7nO5SDywimmr&{lJ>r41VSf_EPC2TzGE*T!9cHMkzKHa+4ZzOh)Tl2N5EauK4SMTe+;TJ@FidVWw2~L$ zs|db31OX3M#xK&UxX2~o+6NiBZqd})F1rscecWW%XG`4`n4k)gKPBd`%i=)P>?Xsa ziHQ@V9?wg|LnPUaIumNbpn6lYmKL6i){TKm35yF-WL_!1J;@se+GR6dGaaj?txG}d z#MQ9-a-y&(LDKJAr8(}4n9=#y&&E#1Z^9vn_Mp=hEd6F4+m00q*(|r!31%hemiPXV z0bj2_auM&LXMsyP^aM-ONP$!?-p}`S{J>Xnbq$`0-`6{9Ht`%i4g6Td)uUIQa&3&W zH*0)OiHHFMyuQJQ(Vnv%uC|{0^0Dr)k412MrxwBc1&ui>Zg_{DVP+rq56Dm6k&l9I zl>dOG3nWCyA1MfF2-PZ;SFnFHxL1i_rBMaK2HU?H63<03kz*vNY3k}MiyevO@lo`v z(G~;|MMl2l8R;47pioGUCe^O$eapk4MFFGigkDBLSAi=xf4lo-W>x2aGjF=X*JFom zNBnMTGnu{5BjoKj0aPBssMW$xy^Q4X_7n21J!edY3MD<~Wr!XIZYgB&eVR1i=Iu zLxgG=EXD>`RV1aqzIxHp&^7kFJ~3m>OHzHFhkgUWoCUqxP{f40p4YG&PdjU8{#%_Ik9n$xP$954G6$|nfhJhXY~VmB1_CPm2)o}$USleL-d z$2Q-lPYWjHajdpUYzzAZTj zlBlm@C-7Ts-!QWOl}^bTe}NbSy9&HJ;W*N@wG1TxgGEb6S!xV}iFV9@JOlSz6m5;ow?{%-^ukfJxwPAnFd*}wrxE@+-h+L;*19ZYc9d?149%a`)Lm( zsW!^8S}(jlLhsja7$ZVWp52f7HR;9XFV_cLAH(bRa$bB4UAH`QaoNSj5qgxN#JgV? zKI%isFn)(1;~QBvpQT*?7T+oPeO9~p&_(z<+I`(rL;J@f8eyQt_Xyf^8`<~I_gGH) zRkE28-H*qbW~(PDb@q9TZ`_~e;V-YgUPC@UGhNkL(b&9V5sPr2T6>yOqk3BKkC#lC zQ|+y{?!|^NUd&6Zl=$lJ_2>82YaRLFT)rRT{LRk&Hz-pV3DPC*f|x@3!a4hihlKPP z5a#__yL6F2iUUR9J#S=RC%a21Mm~-zwjJ_ju3wd~65G9gc+u zQTiOd@VoFqAiZDBoHuX1oS> zPJ&Mgw)HyU$#GoP^oqipN}k~0{1Cv-D%KhFec4TQ*7wHI@Wc-(7>~v+a^M{wH z4TT2pL_}fOmZY7v07ua1rNgM9HG)zMq%NlZ`kP#o^2Ll1FA`OaN#}GwDvj={)o=`- zN_~>?8-kX3+qRiHs$!5$xn$Q|$Si!_Fi3{fHK1HdrDPmck?haUfeh7Vg&h+rOkvYA zVm)kR7)P5*R2!9Yu0-4bY3vIKBo0An@+XL(SjMD zcPfynWv~R7i46nB$n9A8tDLXqx0lg~q|Nr^Emo{^V|BLszH7s%$)?WE$((p~Xn#K` zdL@`wwLs#F((j*)oTUFiIfreZ5Kp~rDcUiIJ7*tCuxehJkv$nj+RT~Kr#L5Rw4Z%> zNlndXl3r6|fAKS8SSBc$%-qYD&xufx=nBnn;iTd&Z+$y`_dQ2G-@+<-uI}GS8Vrci z&THZT=0@ksJg{b$J1LGMgRlZ3D4xAtUPd*{4t~Dp)`d}H|800A!LqYL$5I-Ex2}UO zUhUYpcyQt1chX9P0aWNB{kMBTxJz;y4eqU|e8E|Ms%4{KuXx*9#U0TgRzt`E+}QaB zlV1tNrUF7lHkt7@-uv<7MP|nlp!2sg;vq0nz_^UOJDpT)Hnnw(dS({ZHASAf{_D8! zD!M%134tyUTU2Qr)=>&;=9r#aX89x_7~w43Ob_)N@nIj)FG7M}2oTE?fQTEZaX)#t9uGZ-Me z43NsNdIj708$Viyb`mrT?8xLMf(shP8@Bsm!Xtk^zMs5=T9X4l@oC@pzLLTHiS-H+ zTK(r5+xCQIVdBXsbR3%CXg|3V?*57Q@?BDT=Zo&A!j28jP&5|PXVGSp6$kZG0^eOp z>YBGf4yr!iKfFp$N_GkWP14Qc4!*me4`Ht(OYbA+Z$mZfqoT05#4WfSwrbM`ZAt4s z&0Y&q>RF@sz+av-j@aWENcNa-G5<0;BfU@=kd2yIuwN1VQn976m_NcDZg%;^+{oSO zI=oYY=Gk~?PCPY}VPWQ}Or#0@MYRkC0+d|~HV3lHgj8t?EEYgYV&&PA+ot*4XjshO z!e>zI=&zYxp!mT~KuimH?*phT8_DG0+H8gL)1z2uC8 z)TLsCyedP&Wx|*wn?Zs*g}_68TlhHlJh$J$o6z=&^_n@RLKl39&{k@^bV2zmO2_t? z$>Lt14URf&kE^VSz&V!T=-XD2jy_I!bHT9Wkkemj5Wogl6OIM`t!t-ZMcIhNC=tj( zxaz5yP6tK6FE;Y?Uj?E23L)0gB~U}K4ot-o*-d{F=5jn;Cf2qejWi)XJxu9j^t#&G zu)sudhYUu~DNRhxJaiW(OalG=44PCj7pA2){iQS>GM>;fhskC`4h}u(IS>-f?)(=+ zUW{@MReDE@`34*g>ixCs=9dl%?;V|}*x;lm|E2#b_j{u+dx+pf zA!t$_hmr4OP1B!p*8Xs{Y8?=lR_6GrH6xX#oVyL==FRsbwnl&NUq`v*?;xOHXP6@uC0X;pZ?6FElyYMg_aN`7)_pV( z5fx&jzth(WzJB+bb-ah^28!(_H2gcR2==!(O8y^j&$K}4MkHItw=0S;2rh3Ug5V)h zb_)r9#)7^aMV0SPISsPC__;5>eC6~PSGdnsVB(Og6GIYE zWkHOZ^v4^dPNZ}{Qah3bL#AS9I{@42!`+e(hEk}0t}qvfUyOdhk$+T7zU?{LBXlHZ zi=(vQ75NH54q9v(rD}B>LvtS!TdlfwLLU=o<&gS);rrmXK{A7LgTv~P zO)Ey|;AMfNd67nx(-X_TR)?DqdW*3NwD-OW8#FBzI~JcAFHPw+N|`|EH|=&5+39{H zDLpcmp(#$O7>e}iBb=YelE4*?sAm{aZFyKq}D3Qbv|;_ z2q0u7gD?zoeGkpIZx@jR80L%8fo1tgNJndH$>E6(HNq+@sSw4#Lpk3$?leo_?`kdv2f6~1Z*|yiiPK~bwyudn~2j_M));{OraxK(VfrU!Z z$W$r&zl>s8$LBvC?BN?fpbT|R$VWeJ_`MemAz6{$$=wqHAiCi8Iz2j5H@1HFp$F?z zPo@_lVqSSu&)hiEm^Uq0N+L!`iXioaJSmCk)?1?ql_>V)`d-pWSaQt#pa}Ks;MC+3 z_F=p9?HSY@G^2>{9bO6$ilE6DOYjDD7BL)4t2t67ihIHVDl${`~%i{MPFpOqsg};O6FwG1m>`}-Xdf*6MP;G$D&z#X zWfE;Sbkn~z^t;sKh+ef(!LR%NEO#06_s9yY`v-IH-TA^Ats>ez^n34&6n5dgC2~0; zb^9gw$OtU@~gF1C5Z^LdX+2xoVYr<5^khY0%G%uf&jLr74 zmX4hfry^}T^^mb8TDy7e*_tQ%FO|BkPn)faxDfFmuF>)(@((|#@sHoGmD{xJw#jiG z3S5*>IgO8VJT|T^ERxYTdfAKx?0}7mehGuc<0-05m;}^-H#4GRLz?iQEhd9xT;Nv%*x4x>G9Z-Mu@J}Uv9Uq@C-$Tjis`RxCW6X z^<0j>7B?GAL4(|)G4#h!tcUmTlQPMV(`Ktu_1yG!I2Tw>_k}!wx9qjmmdfgL;73HV z!aXs_K~9lA?;9^r$v2~vPRNm*2EB{{e6TCMWP@0sFcQFvs4o)>apjid%*K5o;rpy! z8rNF1zIhNy~;fJd0by0 zCU@i70iD(lWy+Wv5i1Nfb1Gb1quKrrr+BnitY2AIr<#r%=mL=7ZT>ve_;fo2tveY{ z`>qIczrEqZ`zMlU19$fA3#}IWF`J#Y41>QR-#rQCVusO?YR2Gx>A8HGSwZ%>^j>TC z_RO;i_f?Xlwb}+Fw+^P^2;l+%AbaxRqmqa|(>*hpL*^7%9%aGZ{V^KUztK0%4TN(0 zNZopNnD}>g0@csEule~KFMJNyngv3>M`f(@JWBFkC0}^IZ8_6zbQj+EejLkBz_v~D z3W>m=up_;@Z~EAOcxQYdpCZTdgJwuU2|F-8vHhYVx&KbxoqUQVkYNYE1i&epp!Z~b z#aCr_jZj}C`kCdXb`5VJq{7qBGAC)s6r?TK7&1}djch1F52m*WXkOS-O8k;dV0snlknGK(C->r0w?hSer9BMQ4hPE!D~8zJN`o7kFc-T=4-@@X&9*T5ED<(*#7IU`Rq&`0HGm{MPoik5H z_=*9$IKbTtlfq-sp^O@FVMWI2BO|cxNloFB4MZg~`HI6dJAhJ)Xukogitx0u;p(~J zedDSX zDvJ)4mwe;m{X$SOfS&<0epOms<9kDfM}uUA`%E)=^6+(V&-g%fH>vZs2R`JRUF;zv zm`{_#WX%}XTsH%tRc-l^*}tAtn9wM0S17j2okP-%37@Wn8Wgs4J8>F2hZB(krUh#@ zT|0v|LGeTDbsTf7pj$L zEdeKLBcxKA;d|)BHMtB@p_R|~-SSy2{p;g-zjCO0XBjIC;NN_Israa_Z6wiMTd3(X z76es;3O=J_`58Wd+NYP&}%(mXza}RR@^@VD6(ox?I2@uuD`dcH1C063V#ne@C&90 zr>ba3oS=d~1;}T2cwb>*7(K*;=>?e-`1HJvFa=G}hFVf$6*J zM+7!AG5~op8R`5vJgvWRRF-O9W5YRTzMjYR{JKjaj5pYz;YTNKkc?tzS6SJt?t{B~ zo78Y(vOIHF(p&Hj>gxU9^|t%@NpLDeJSWWF@xR2Dfo`N(0AI@B^oZ}0D$@k%au<|+G(Nj@T& z#%XV~6h*qvT5wOVpzGz{Jq;De#=|plB|5}((BN^0v$Y)?;fG$wA^zIth3yV9jE`tdkXp8wErno96L@2j(+FW290 zgL{(Fu8Mx4qEpCcZ|-Zu<&}>+z<}pnLJb!L#ks$_CDZ8P-{_jGu*IU#rb#yGv@_v7 z-@t$kL#S*Ctog2T2^MOa(;Z*9P(9$VtG>$Eq)02b|dA>ENOCMRC zFzrFf3ib-Hfi31#E9#K+Q>CX4{1enqZZS)R=UoE^K*k!_lEOxDgIT*(L zD?=kl`X2j}E7eepUG$Odq3EWSBeOeUfasI-kNBrCxYMaz1#G52nisPqV$`8fb+-V( z18FY@TJn8I#v^SK3Q;*I$s2L$n1~g8lZ$cHm$EoG`*maPFz3(35VZy-xV0pHg@LW-NNP(mDh4ESYtQ zCjgUq;TDR)TNAx?HwPm3Ww4Ho+L^4bkPPkb2<2hF*b$`tD%1aHh-m{NFL4g54O9KM ze*U#tL*d4m0n&3#QKlfh-Qr#&pU)VRozCW%gp*)2K33hM1Ktp1fe+>+6c3SKE_8ok z&pVf9>7o>Z6yjmR>VEb=+Mm;#73&Zlct6_C@l{^!WAjxC;b#p{LE(k{ku+~Zq?sjH z%=>`K-6beBdP~THshz_w-n&}GOPZB?hc#4~c636`A0)FFuuWsMq>eYu+2~{r!kgvR z3a7CD3Yg_`Lje6@RuFVWC<=P=kt&G95HXAa>;Vc3>cl&ng6dT4)mwZu`44eFFwZqbl-ox@ve|=4`u{5Z1!ER$&l^ z$g^-fQb}%Z19@oI4n>Kvp9@9oXGRHkHX(8?xh<+rtzrYPc9DT`8Z$YHWJ8ICqglPU zo(*<2aAtv_`IB?GFX6;U+{iwo^n;wC#9#HiHQ?bxSROkaYbnU_Ycc=p*TW+UP}=ES zrfzco-kK`eW#%8E?Ng>fXp9XTEnXldGHovHLtr+L9my270gsb zGXF0EB6C{m0J@>6wb?Li-gV}a(Du)L(QPp^FWB3!-pRLd_rV?C2sO8F| z5DzpeN}V+a_%(y%&_2kg^OwS9NVFxD)Lgw1G})n|J=|IVsBwLI*1_5J|D)*|qa$6qZfta%i8)Co zwrxyo+qP|EVoxT?Ol;fc#5Ov%^Yy*={eC{r>Rw%|s;lbkbN1e+A6+%!asm-N-;3;N z_kH9h7*AUvc99iE1{15H=54m+tE3XRZU}miQzZ7Bp&I-?QP6(xHW{1GNCN|2{O}%=NXzOG| zbs|>$1~?;K#hJUhDe(0)!?oZ?u12AYU|K{B+W;uv@%~FtK!qu&4?T+g!asKl4%cWr z&pT#qf)qr=Zd+L6%b+ci?~?10fgVcZVE3Bt)y{Xf1-&V#ATEK?R{k!iSup@dxhds#yH-4t-!Ml;xAMbLhak!fK zh3KEZVtWGkv7-xCA}40>{RJa)MsMpEl(hPK(ZWsITYtvsy>+fsDw$95mtmi06bY9Y zrmG7>Fw%6u^xK6n3rk^s=z5L@n!w_>Nb}FdwSoL2j zHVn*%ITiqS^ff8710PkG%RF=*fHC_YqQHdoBqeDSat}u9^pUBl0DdyEh8cckXt+yJR3%{WiuoDF{&R&vHrxEvu#y~2Cm|a*G&pX zF~goFCOPRI%9H#>nkke-<-CY=9pJB$UpgG;1{W&ZJLhn?p@~>+MXl<>$ML?T2c!D} zYP06|{_>TiX7I6Z|OE@3qT@{(!&PW+Xh)!q}RWK=hw-dEa>8GbQNbZ>l#38`x9;5R`bK084MXPiF_4o&wiH1ri$nn95w4gN_VLK&Tp2JqDG$ zc96S}1*|=Xr?JQ1v1FF&_suFhe0Y-}CWLFgs>$T#LG}uG1M|N5NZl`U?5v#ka-TU^?c0_4yx=`J<(%%9j?+?BKIH{fNp;&9 zUgHPO*Pgu&q4o}STZ+6RIdK)+MvwCvoJQdw=?~tk9+kV!_Z~~`8Qjy;c|J)dGXR|@aagw zt@X>9TB41%sQ9L)+;;aEc{QESE!Wq`b+$`WdaDg}t5&`)2a%5U|2wG&VWy<&CPPONW5V!AE8NN# zA7)W?&HP^P>e5b57#wkkXc|D(h~=Bq@?W|Qtf6h-@2@L=X;kD9k9vdvyuMF0^ms!$Li7d{cTmTXa{A@&qk8SA7=U!tW_#i zsdPaEwd}^^tH|xhx*n>rKnAy*3ccn{^J%Zigakb935I;;eh`_@CgJly8u-&~rn~MY zw0X<9{{CV^Hm!b98Kul_db1P6Qe$i-bC&D7Q`6A3e}B1mb57u- zAz9bFOtHOArsVMWfPh~+pQZ*}A7dp5)d>CWx_&`jEdYn~JuQ~E3LKB_R{lVsTawzZ|11?6=mx5rcAqoL2;AC5%y3%L67&h2`fIO%`S zzSiVddbK$rfa{^XwvKaYK8C^|IVo;FF!m#P5bz#<@d6BwO+Y}YrW30V#YF()YnQ_h$keWWYRbkD&?`PlON=N1hYZZKAm| zL0i~kI`*8(XJG&r&fMu0E?dYRZUjjVuCd5!ztiK}P}~>&r7PAz^8Tb)Ch6cPjnwNM z#HbtGJ?P@W>C~+m8OtyFW_aPL^WoU<;a}a(vyNb5>gtpUr5#_8Cy_cHd$ofQJ0msb z`WkEF36**5Tc5uIGxHg}pKaDkwm=lW6no6)lZ{91vc+7A5B!Pe*x7jQ-14x;K(||d zU055qxM1CBo?c<8BK)e9+Z^&6;g~BQfCFD>DJ7z0oZi;5-}kg$x=j}%L4^y(IN%f? zt;ODkQ|YteSrIX^DB4n6lH$?!EVN|wfFNTtbCKlAC6Vt8hux+%VO>21d5){^Q`rAh zFGaX51q1+nYq$Wndw85Hfsa~VyPN%}DMtVdDh{S*^Tozlla52vI76nr8G4S57%oU? zQmJscX@8G(Uaty&m&f%lAG`|k!AErUbJkK$mN|)c)fxa4p44;_dDq!D#1@R*uKPWy>@9hur}z(ajF77 z9ELPu_Em!U6iMYRaEAFpj~9Ls;GV%T;c@p{$-VTD$$G%CJHqM@O_dZ1h?n;=7M#9S z6oE{{rWh@nnlR{Dtjy-nbpk^pq0z$9Zu4kQn@qCR>>&7hoa34oSt8Bw4{y^^G zyDSmorK&;fA#2!q#cDQ3Vc?OPH;kdYedBYuMiS@rt7h4$-(~Fbqi-b;{C!;(ysBj@ z1O}y{V_lbq6HEP@!y4tbK7y_X4OavTVbi&H-nH`gOg?F@w@A!=j+>EO%q}3EV0Zh$ zp}SMbG0Gk=9k`c#l1IYChdGBL-5z|Y1F*LSz7SmyvQ zbx-?fDjJt&+ z%px4P0|Jw-*kNrEJZ&0snQ?6TbM@H-_`RW(I(A?V2NO&*4sZw0Fa$AHW+EE~$&!Cl z56OgDL>Drj^sIqH;^_Kxu36Uv95JJ?=R9!gJ^_1ZU3UCi2hLuQ&tcI4H~%420~#t1 z0yJ5U+>@t46vPOfp=GiIxPE=zJl~_lqpD8= ziX+@xvRkg{^{1oND!;>af0Byw1O0f-Xd7{VI7%Z=Hh%jb7){yXX$e+b455 zr!SrtAQz>LtRsp+lN$4B4hf4=W0gX+Y;%jV%C2E>K86qUG<$BWL$5 zfC=|}ryu%Zu-dPmXIxZ;X2;@Y;M%?Y$05A80L};`E`sXoBXJF*-+A^TC6&&Vd@H(p zN{5XKyMC=Bm(F7npxR}(a}*yv7Zk{E_JCbNiTJKHWir5#g~CY0^|V0hQFfpOHT`PP zz0?RdGVEijcen&E|8evMZ`vpEj|b{r3}U%9QLs~1M*?mkfCa4tesHiyQCr{MPuv0i zlRgprOjm@P?YB76RAIm5di$!%P$+03Mn(JwPW%|J**sVBI$y=`u{c)i!scE3w5&SK zTxI0tQYz>~!cVG^} z^X*^_yEM4YI+to}V-TbJJ)`JnxDbEQTBVJJdr|Ri(DeM#{6)mAssyM+R^aoQ-+2o* zQ6}*C*}sX8j-H&@yXiyAlKPFo@lvbuys=WR>G7edv$N9fvU|R1(XP@b5NQQp^??XH z84R8kp7yNp^YVzBd!G{ON7Kcv%1zhhm+tGgDX)iRHO1}V@A$Jn$R*>s7BQBe^Lj>qx2&-N$cNnyi~B5;mB4-O$pH$HT^ z^5z8y)6hrhud;B`tQW(Xq5_MB^;(A==!W2M;Kb?0ZEAyUQ`5nMk%5O@r>prgsW#69 zc(OE_3N1g5z+U>iuiy*=)J^U0EKoIX#Nh9gVn_hO+bZEbUhpKASXcK7ILQ*_$j;su zVd;ERVDz3QtvugxZqt7JuuRwWt#wOldc4;#a4N~>BnlU3)S~r3gXrb^SQo4)1B~>< zFrcR_Z47-ZC7k(eeFilE?`}pNj4a_uIXT;CZV0O;VAA9j}HS;90dUZ7N^7Cb0CQB?{^-0-Vi)4o8oLV zw5m7#=!>OuCNr4l|7jpCzHFjiS%p#Oc!nTy*g{eR11CAc6j9Fuv;ZZQFYcjH>ChG$ z9;XFvgIRZ4+SwIku?m+-$rk?_vZzZlm8sRI6&II`b#t#oA*T!z#yl4!`FeafYoTh# zC?(Vn%BEZMQ{@6UWo*liB$%1&Bzh=pw;E>Wk{KF8Tf&_gv*|c1=+Lquo6Ix{G4j~r zHY66~X;=@J^_thIHPziT8eY($RQAc3txoBH6vA-3Q9kRVe$N*d5rz@%tC;+VmEsMS zOhSxZH_FbCeB(fFsZ=*ekDhSkvC;1~X_syw4|sr$F6M^yS6p&m&+VcM zAIUy=o9_3;?2Vrx+fbx&+E78$^iZJ~p5f){zpXvCyh^(x5*^JPe?GQvy|Pp5S9LkJ+^4zi zoLJ;FvS;F-eY{>c5ugxSZM3Dfvwu9-4(F~6Bb@u1GYc7U)4~tiZFS98br`p-)A?wt ztEXWqolK1clT&@qJiXwaWbp*a6Ky0Dz5s266Q$&0({(Nu2Dv(N){P(6OgC=#$E!ij z&pYQqCRE#x!Z^gwgN!-qodVt!8$L&Ni^VQf@GJ`ik`C$HO!jY{9=sAN`4l;!vNjzR zV-qghfiXGp1%&g1BNuUJ7w{}gZ*CM@7CXfe`Vce{2LrgClxgE0A>Ptmabu&dq3Gd(2E;blkH@t!LP10$X z-Tc=Ld#~jUf_?|!%rBM99NuqY_zaKhMmVVVM?q#mK&PldJ>1{AZi5GlttwQ50PU;=yh`HU=Ii{0o%29EPtJxC-$=x?!}#0!XTxw&iP+BM0>PMW5WW zNVi~wo>k;zCQ{@vsFto3-XL!9ITYgje!4qxgP=u)hv+^~v^@q&Lj|)M^-*YvA@HI6 z^ztqv4q4e~VP4x4o9X&Iro++B4jUbY`=VwR*xd{n?i^VL&!n~Hd- zG?eo-?LtG50uOchc*L}d%jy7Mu?wBt#%W|-pZjpbX6t)xn=%UuTHaw4nWC`r#p#cu zfb~ZBn7?D}K-<3=PR_00YW5|Ne@(9KLwtpDzpScE`vwjtZ2oZ8s$Q&Yb}p*2)gu2KKRU;14khbmZhf5}dvGwzbf^LYb*3dwrn=$Y1CMm9n zmG29$VfWZ}+`&A|fz(ofCrN$pTH`K;>qm<=T$NpD;3;8?`97R%nyhKix~<`&lH~~{ z78k^Mz`>ZBu2o<%EROyUBP**v>Y|=dzC?QsJMgi zG@G}_yY{MfgBxzMQ#P94>Rwv|31)UvjjL|fqiT0}I0D!Vx~8lalg=@{X2!?W1bjUp zbv-@sgKU^a$80d4%`3V+3I(5Mgg%#P5{{Nuy$Vw|nI9YcQDu=^q$(q>I77S6HgghS z6895=6t0|JhP2{zI8*@-UV|;&*tB+c*?cl@?+GH^)R^)ys@A=kVVI+oe*GGS>q}wS zwPUJy6qnI11@k#K!;CI@Z!`tt_GZlB8EeEpJ`C%vE{)Y{wlh88bv>NeRcj|sk$TH&*@-fHm7`bzNqPQ`s2m=AYdID&Fbxnf@M(@;AveL^?LA3hhOz#3&|; z=C*V!q%+a1Tq$w`a%x^)=ut4~GiWP~Z=&(%(HsDlF@?({n;Dwa`sf*IT8?*jb6fp8 z1raq+&kb#=!jrs zBC=wL-(P-xHAOiJ5hD?V5_cx{WA(t5p@U>f&2Z5G<+>&ZYB;bU*6S#jK!zT>_Pn_~ z?v6Mm&@pM>0=48irYWV6Cpt1J^LVUZlm zr4))y4gYN=gTYyDLWsxWS>vX*4*SWZ=a8D+6AXQ;P?u4a#GK37cQEY@sb9n}8Lz7dQx7 zjl8sQ#E|&FsZ5p7*LATsETFr>Ami;r5%vZfI{TKFN*Q)O?UDwosTinZF!S_SbT$-x zFJ1{{A`je-mjt7Hg%5n=PAP~4L>2#T$^h5p=hBxA-bx_$?)ie22X1V0I+APn68puO znajEndbq|5#DIU^4%uEQ|HXWaK~IsTzpDrsnLf%yd5sq!hx5fI0h1T+ff4LAM1Q>> z9?q++Mr!TJzrC-s?_l+4q6=s~<__5U_(;Gx%Y}|PFTm9BSZ7j3`jE0H*}{7_XU=0l z8}oWBprE$<@@UH-l<|4%<6 zi^mXRUjwe1pKh0%3me})w?5y8ydOJzu6uzFT~^cGJgX0v6Zfxqo;T7SrQ%xiGGqB) z`(iG~%VncF|2$N$M;CG*;A z__&>k0k=Bq`y~yzi=22pv$`DN-$xmVYEOScLK{B?ypHET^C?q+fo-+c(Ir-xJj9-C zPWKDGCACXq6g^G#nRTVZiXBPBeK9Fdqi_5p)5ao)jSBTGG-6e`%*<}0ZVRU$zA(ii z2)C^f$g&ZJ*ZiBG_jaA%TEoA~(EzKPQ{{2zTa3-wx)$phPPgyC^|6gZhgHBcus-c? zL+#^rOnsGIc_>Gty}L(7(7Wb; zO4f}TJlJSCT%oRRAZ5feB3_x%0Vf)$t+Wl1UU3jS>o^g$yRwR=~B598LVdX+h zAjIM_qptTlR;*8QWhOe_Ux1|NHYONmJ@_ZM#;=g}X6{YM>Vo$K1!)gtPk;ZQEB$yk z)e3wDJ{!|fI&_us*FQBT225NjkD=Ia$WR2w)66@>jwl!WE#Mx&A^ORe|-g`i)-eoud(YT)jmSauIz>oIY3xDoBAC;)o_N(O5yCxXw3_0G~ZN zg?G|l)?Ew_5Vp~dDjhT1w^7Ft;&|gBrJq=uenp1ILWi11RPF3ltpi@z_vv5Xa0qJh z>TwdxEb$7}uT?7oiK8lnXZrD*%D_LPkKf0g_lBDj4pOY&;1Kz`cOjCFUh(J>Qq9A`%-$LqH~ADTgCGzy$S%|FbK*S5T8SigPG;?Y~p$3vHyBQ!GR z85WO$*&!e4ET%(Z+L(CGh@?)742!|>AwXHu^FLu~S6#%xJt)1)_7ak^7!MV^?G#^R zp&x`!k_ma#vprBYujxsD-jVBi{=FK*@>mA zCnuJ0QdK}mt&ALOZ^_?jl1b(PtYBo}i>3Amz0z0~O=mz|4Rs!EoM>O&Ce!TNa(uyI7+)=CL-!@x}06wsfQMLW%~0-M-y2wq+AAfZI{xo_`zhupxq&c_nLL%R|P6 z$+OxX!UJ$Pi943SmXAmDZumrhhyk9?L;@c}Ja{c!@|LPe2HmCw-Up7&Tf}o(+A(Qx z)`Pzhgd$=+MFwb!_3(CRDAS*06iE$V670P@K`2k?yz=t|n?Vht%3%HAIjxif3zO_u zXbaMUO}`Egzb&ZV8xDREpvGpS0WpyUoxd|h2+Sh!;QB;t`amKj?PLKTcd^YRH zMC5L037;hzZk$N1Rmf31q_2zv1`%@QUcxc?&f(54slENd$3jPkx@UE`2#GHkfDA`T zH^%jX9SjK0gr#bsa&4|8qlCNbSpvz?*BTnC1s(H!xVD1=Eg(Ggb8oxXjrQyserWN! z%Xp?R9bWtY)H3z;-J4Xp5WhOx`ZS^_2=Xr}`9`F$0@FLq*!fnrp11VH>IFFN=lxXE z`+-EKFY@he^&&j%mDMLXrCyp}U{?NGKn5ZELT!Y%@GNSriR;5uYl(2gh4NTUnT z&u~v1MZ#TlqEt%ETjGrc%`*tY{t?~oFcuPm`P(}(HRZIF)zF}d&$2C|^L4=a?AS>m zz*`83a)J{MLN?fys&oXxUIVyPao2M?ac}uizGi5p<5(}EqQ^+8b&aHjf|+5iJdi9N zT7iF?UR9@l!`k^-{-~({{WobTn^|b_=&f`!<5AtpX5=uahFPU#AHV9$jvODmR2sbj zrHZ?x!nn*~)jv9UKU3z`y=lZ?ay!~gMO;KOgxwn2arQrr+8gi)nUwJAaZ`e6|-uWByWP zZg1hTk8LzWw)Zyhe{OgYwLD!tcI^rL4I}e4QIgMV2(-z4-sXW*q@1+5O`GhJ$|Gj`m5Ejr0$eIJ`7@0tCUr(tUTCvWtiu7k}voiiD!$5o6zxaWuvLesxo_BRziF|Koq zwOFB(#1kXn0@?z=8hOt)QqZ81iP{Y8RB+S=bNR;FvRO$If{o|t_z_U`kIN@Vpy-Y^-8^_iQI6hwCG}g0J{yqa3#--k$wEm*wlG^X=rrg3sl>7~XanLUiTw z*HxIR;SG0?``5)Bl72O9x#+PslKqfKh%$wI1GJ~zYVr|t@HXDM|B{)5d$GjvCczZ> zn=7e=jj`2$PK8!$@7}{=h2mE+jv%!gW>)!A074&KAlOs3dS(x3;2LwEb++2 z^)jYr3i_%lTl4Y$;-%C&F@ekxFpF-pQ8h>drEof*ZQr_}XJYs<6T58_GkQ-54e6e! za0aqq+Skbb^S6=kOu+0<)*rTFCXdI>#SIt~g6%JV7R}7Zpmes5oOSU?5@l>d+cM2MPw6}&yt?eqqMHs)v>vnQZ{|Ho~7R7e0DtK{^r)h zr);^QfeoG-ehN&(6`X>A^@)!RJk}EhK&-`6GL>UfiRmdn+pcQH6NHpz)l;Iso(iOEiO6+~IJGmrzLZ*m_VRTW1i zZ#uA`!5tzOz3wk_&MS4*XZ8&in4Hz-97ih90wzafS4P z%krv@xe>)aKDab0HS#Y%nYwRv36hFqgsudlUkhL+*vuaKjGb=XG^ksgB2{v~YZGS) z1;21gZ5Ieg{nVXV9B&M=)wQCL-h7lt`#SeP<_kCyjsA{X=&`9|NVwMQVQ_iL_vDt% zZV^AsJT)_`_eb6PWODQkrA+duD*8=cC83`oOuZrqyB_nMGSvHq<* zZc$7*ST+L1{hVo!ecaNVP^<#cl%6!B(cEoNB{$gVjuN!HaUiyXU+xdWDWQ?Hs=UAP zb0I_=7#rgsb9KE8xa^pZ@yhGsToPL?*OFB04ca*x1AvBlmZoz(k2wQf4Tt| zbz(}S2W`^p7-7E7pcslAN_C(v6Vs^(zMi0Vz+EPM-1^AZ_q4~TSE;CXUC^!Q^@i*5 ztZy8@+@F_|pGxI|)6W%!EDgyNirrvTw8BLX>;WL1AN4#u!8e^8y6wdK)@ERSu*2oi zE93h-)cX7R@DgBy(2ko~rY1Si14m0~M7hOq$9Q}*$Hq>Fqdv7R%LuD=H+si@fs523 ztT+0aJP7_mjV*1wBZF|zfxDTRcG$#8YVYIRVe9hggR{GBwQ}8)??(#zpu-K(PeC*& zGB{st02d>8XgcAij&qwrHn;3RQ5NooK-nMWzD!m+Y!bP|e*7Yd_Ey|*EEt>txDaF! zaGT$>-4=Pe)TVEJ948#9C z29*%Yg&-n}AL_o6+J+6m&Of_CQ<+pA&(;@JU{vO6GUzq+y`bXmzboT}c$r5t{oJ_N zsBe$UXZms?aUtKU%Z!l{_RH)R$?RV0Q8NVDSjxxsWP7h6$ws9qL+@s+IR(pX{~&~r zG=#~C8RBb25O>gT_*EZS8kz_s!owg}O|%gh_TK@hRCDw?Y${g`Y?sl^fmU5gS`GTd z?E0Z~Nw}X~?zeHAZ9Sk{@Ev`GUvxzjJ$XJyD48Wax(1vM3h5lWCXdCf?ENMnz@!pW z3^kcyqlHM5U|t{4-N!#57Fq=>5Am0fLG`bE5N`F4g+ia|O?%@b-y?f|UGhcgV5)&v z5CcF{WQEJ(MpDP2Gp81eDH}ZY!wv;2Ps!zE#J(o>@KBo zzXEYwDC)5X;wM<}>p0rPp?7_x{2WFXh#G8w_F&m&T2>2W{ZTBnGS#EXAz%@XJubug zsYD=%nF~u^v2K);X*VO#9o!AEbW^Y*Xdpe4p*EYHM^n#rB{&bm1tr_w3M+TI#`3 z>tuQj+LtNiZ-#~;Hy{sI+1m86;1YI5-3~ENK6&g9aN!$>-Uv4gYQgWdLG){`S0ADt z0*4{Wao#k0jRqRD{mYK6!y=`QRO4$|RD+5?)n>Y27HhV@PzR~s{ReWBFeuQhxHIEnXORgL8~W8ybE^w zoe%ax!SZM}UhM}M#I9suuL2RvoE8m?%8w?%K=eZvn@>jm-r!XufE+~`$|@vKfnLNQ zgE-Xe{UvTuGc=*kRgTaVG8tmD8mDh4F^Cezyw0zkgUCjRKy;zZG!~)Zts*K`vN9|K zzAdD7;FFU+wV}L`KQJ-S9zjWoE@~Mpkq-jvO5`_LI5?|M9UI>hGO>!TunRmLKqR8{ zt4Iy0cy1v4E!9<<+4!Nr5;8s}7xg)}O_=6xa(K2{*OKS1gQn+hQXG88-Uq(nzyX3_v!v`L>=omBvj#~OB8W@9tU}J?Ago-o{ z*1u%oOLY-;cYHMow8cB~%C6rXId3J)C38_C7lzJkCvOv3PkupuEjj+Z2s2-&988)m zAp!|UItnIrpOKb!h)rH6Wi(|_Y7wkDP!W;T!Xd6L1;x@(*@Hk&pw0&IE$UBUfMJ|D zlkefOs4ntvy?z1aX^`8pH?bD_lV`XUpP-$NEPi<-fB>Ex9CF~p>=&&TxR~lKnf|X3 z2*ujKnGB+PD6TGllQhH;OV;VM?k$IZ@hyQci6^XJ%?hi&L(HyAC-?$KYhM<+I^yMB(b9-9dDVY$}U*q$L=J~I%4p&iAL$dkc^ zYYA(w0A(>h@*>GRfIXS@s$2779A;0zsPV@Q$mhoU1hZ^jwY7cE=h#UW1$&Kg-FNh* z&k9^cPj0`%;(fLc?FR_Mu`?iGTPN=C+di>R?3L3=-UkhM zs5~xIZy-)?1!;!amy&Q?hKVaCbIT-Q3MM;CM?;j22n|(miWqlzT7=;l>sCjIIm|vG zTD{*sGrIlEdz%=HvM@;eivVL`I6RnZR*7Xzx^7|XYT6Iu96IdS5P(oWnW(+nTsw>l zQp5DVuw=F<6GtSgZ3PB0g1HA&JX0RT62#%5&BD4N#UE{b9TQT|q?@Gh8t5EDMJ)wU z=1WVbNO%&%_(oQXQ$IigxJiNmkU6)gTbpdmGD^Oe>P$hZt-vwir5t!AG>$4w592BGQ`X#+wFHJ4P5-AX1L1e zM!1wHfNNe<=-iEiis34r&ePN8NDNmF6bb#)gRd)FQu$5)h;qacollhM8Ls1X|5jK2 zVaJK$-rC|<#oFdN z2&0muO!z{$M}@;7!J(f)w^p%H*qw^QM!q)D;Pw>ZUvwusda z!?`e{Vc;6;2WyicXGUZe1OJ{vrt*}EWQ{z42nwz}YYb+NH`)^9LY$EC+j<5*Y7Uoe zI`@thpn~8aKC$7hjW*p$IcGvXcT|gTcZv3=YYW@gDm`Z+wszW(~nD<(+BiR0Snh2 zxq|HuuFVv5q1G>`F0Xh;Sc-CIci$Rc9j*xwK+~V0n1cYoOzWO7_jS?&xlBU$U|y9v zNG{dh)2`33@}mVn5xPBTPknr{d7|;TqZ)*yTrS6?39{j?7u8+(^NXjX%;v#%F7-PD z^d-|0>?O%1^04CFm2>zd2^SQtn<#U=Da_p?hwn*@0l&X9qn!Oz$#?r)^N#uJ-IEr+ zGeLL#((Y+YDZ1WDI=UXl?F5p|;17iz4npX3SdavOZU#>V8P`pl&JO+vy|0IHhWLjA zwPXzk$%s>)e}~m%4+NeF3rZ*wg3weRzvRWQ+niYoyP0|bs*3^Gqj6Z?i_1ZrAr4}- zOK)LCXPAB6MNOE=z=M|anQP7@f9+na#Njvr`@l>S-##cr8|yAf;i51S|+3zq*g^lF0Za72k$s%Rhtb z+6HhDgYS0)&j`{whI9a2`b5S{zV=?v;gwB~y$P1z_W02O1Vp~6dzVmVb>i)0!h_6Ql3Xz2+%JwrwmURm6)-@0893<>Lx0k%F=(sdAO zO4LZ;j1|WLYmqSV0RpdxvF5n{loN5VA@XIxLpR{y@ZlbHMMiMkw*=C{T6@d~c5}*e zU(Z@EVyFJ)6Agw5QNyuS7))N@Qjz+G&X}ID=n2Fy!ch<6pnBBW*NN@g>pYxN1Re|O z=F|j=$+TWYMS%*`pnQ1)=~XM15JXQHHRsz~L+|r5u^`K2z{ew#wlJuaf(S8yNx=Un zReS(U+JHesCWyfv`8e(p*&&uifomt=t1z0TsYMXZjmiPloMU)sG^TfWPN0iL}@`@@95muJ_bsJW#dd21^# z41Ridy#I|ZUmPy5{|Uz&mG=F$p6kd`{=e~Y8Nj`bL%?M!j8vep_(KsTA2Oz$9O;^= z!7Pan-ilJg-TaP_dWekz&W&Nw`j}&hp;eKe+iNCX#sa6E0}+5w+t70g+laCTcNlUk zIVy6nJYt1nRpj)QF)%#p(zozSU+#wa*Aqf9{sxeYBuMx>e>)@+^zREHm%V^QI_Lf|BSOnyw|(0Vi1x=g&9H?z}xz00{l^<@rHP$b7Qr6v@%rk zxjf{5%h-|Ex&OKnEHAw_-m#9Ob&Is9xSK#Qk=uIB1yi$+5#aoS6o&GUH{lg&d@5f3dES=ENb_-eLwsSxfC+ z`leQG{N?*@XCKW3*#vq1FlFMIqV-Ty-}KONS{|7k&w?u|p}<+G(5`AXB|e-<|5e%o&Qj&{*!8z+ zRr6wFx{0gBb6(XqY3{!?{=d3Tw#uzyP!H>a%92&y(}vv2S?lSzYil(mEeNWPh_5s| z{fiLI*vmBB4m-0#5u;_43bU_sP#^Q|B;VMfTNbO=OySGkbaWQ1WD$+wh*kZImF+j< zA=LaUZNBgB@##5nabVd#mX*`yQI-TLhI#t&zmnoinGw|aNVm#x|PK)~?{5+jcl zitV9RNGWxl~u@;_5&+ zy@yhSRPVJvg2wyD=H>1f#4!6lc$MB)p%OM?fI?EsgG0Lo?>arh-ym<8`;&$Fy1F{MlN+odu8A)@MgE=Mi_H$iYB7A~ zuoRf8iHfSKs;1o8%}+dkhS4|

z!gULP(fpX_VRPOH=WBSheq&Nl0pv>PqW?N%Gm zI`FlVpyJ$sOKWpxv=zdgZSMQWN;WG$TPYRkHB&4)&(M&cST5;TQ7h|^GTcjAzOIEg>{+U#^HG4+dd8ccDKgE$!u;Ut)o>ppxt$H zaaV#Sn-+JqVveqN$YUSF~I(4sqD% zpTm#IVKHP{PCSTwo4I9&Nf^VtuImMtZ)8h4)@?hNU!^E z5%+3`UTLH`s=x54)x@Yk^XV%?vulE#(Eqqx`=)!HFZ4%EpR2m3iqI>Sl8R~t=+Hsq zn{VN<;U4qDeBpdB-QfB;3?b9;z4ko^_zJ>gKUtQPeED*U45s@Cw$j%7ad+5#k9aCp zF@t|SIf!fEVr>mT|5r|sg#^Fqm8=Z)&%*{J%Jlrc#31*Yj4ZEg&aefL1;hLwRd3l9 zN7r`k;xyj41b3I#hS1KDf*Um3+y8E3sX z_FTq-=Z9w(&nN~Tj%zmc1I$vGf%`e?i=<=M%+q{_!N?Cwr3`e|!xq#sxUV%(tTdLW zO8Vhq#)iUXms*Ofdf#N_XvS-l`b)M=Gi5Mo-JE@H1>&~jwelwi6+hjy3@hiu9%s4y zf?04(kkw4kKV;EYZ~#tBEsl7z+EXgDThKAlcD-JnwOh*6-K8x%FzZl}1fV{tQPuQw z2&Veamo_MVf&rbQ*c{5^rj|Dhr*w)`$|mPejSXkTng)#zj+hJA73vjhw%pq`Oq7p1 z6L5s}4KB-T{H``EQ2Y~@lS##=Yt~X)4>g;Vb%ahoMm%?vNO&*JXi>wl&F+lz%10sfd)^=%oZrTcR2rhC-% z+*(`vP6zalcVucvzK0%+!~-3`Tl5|39%qzG?uX&nBTSx2l8apff^Fzm^?u=?eN~&W zLLEh(1ffA*Ml>71Lu=ii5}5+F9ZTv3dRR_7_J=b%#eW>Z zGtbEL-RenH!YfJscENAb zdH>tt0b~xjP~m%|!C0;K+jMX*CKlmHNJzGOKgV0Bm>dH!{$CaV_iV-_((_$&u8DVT z&k3T}-;5*hbIyb|kiLB~m8CTV30G&c!?ik-gAdF$&o>KFdVFmy&3oy@B-G1jYn8k? z>~Kq%PUoDA8yq6Xif%iRv)V#pVzJMvf?86dP~9zjglQkR-E#_|+OK9a&B5!#;!aU7 z)=gt}n;t_~qy*LI(m)o(0uK>d9r9zsL>6n*C##7Yk7jNVdre1TZ&pz7-pPXCBV;Jo zA6scw-V5I}v{QpNVLNlIY7AR*Iqs}x9B3j7>{kT_-NMIfir+0`hwj>CH2fu_aFB>y zH@*k3!oPmzu1Y(glpc`xu^7X|*kcqFoJNqO(B|fIv$O~I7aBmix9TNkn*V(927Z%@ z@ttI1>w9ddMZOVSYqw78emKr+a>v#m%iyp|geeeYCd~#K2bY~E`_I>j2B}Fb(ddli z?T;#snFNtGOM~NbWIJdGN6yWy#M;_T9?kDp_LZ!vmn7qB8}HB8S5KBIsm`VyP{^!1 ze(#a>TmrI+wg`%@Cp&?`u9kn3%P$(#wZR17G7@uJA2aAK;}ch#YmtbT-`O!+*u&Li z54pTjLO1XCCLMK1c76D_{0_$3xOOiHTW{wxXhutPY6Ag!`N(G5 zeB2Rb10#pDpfu75fSM+=(JUC`h0bNpq*KhFAdChF^&moG36| zTylS0muxBB1R8n_HCXb^yE?nqht&{gUN|#k(qAYuhCYJ-Hl2Qm*bD~4$6t(lru{5x z^f`>;OPua`Q#fajwf?;)8Zra2!G5U{I3~C5AFfqitrv<~20m{nSpuf9%~rY0m4S?K zPEVeX{OVPC1ESc6L$x`!Vy=lr+Nz=FJ~#TrP<-=8>DS;3Ni4H*TxbJj^8B{p?Qc5jfT( zEB0@?LxUg^iH|VBJ!6KDm4}O9j@Zz9l3E)HOdTON||%`*Y}k^HrFPx zCv@3VBw2sFH7{3R!ur^5Wv||EUixe^r_4d@>NlCSk3=^Pbai8pKS#)LupX{Xg~Ak9kgMA2Hn(s&%}$Q5 znt?Bs6+byOOklI`pw#gFSsi@SC4f=h%vh((C;(F#FEx8Q%Qga(OS<=dtERMO?|b71 zDwemfF0e2GZL~9I1L4KGrZt({wT7qcML?lL`7EltaG`Ip0`dob*~)CiRUI{e%Cbg4C&q9DauyJ z`Z<(Y@h3ii?xU-kOh6`gmfJDoAeY_jX`WSS+e!kUd=l-C+k;{`tb$VLaPzFk= ztUNuFFJOZy@M&1N$Fdvw5ygL*qZ*svX7tJ}9akS!{Plz!Nyx>t8;h9z+qByOqO(MJ z$8*J`{#f5$r@9zCj!SS@KlDo!@vWOEWaS&Cl7R8$xU2hf6)#BpcK`Xb#u16n#d2^Y z#+R@kQqM~wu-|n#)u`@UqY5mhqd^!U27OQLh6NYve15Co^K+3(PGZ`HcT zQW=30oLbm$h)eX}4VVN|$QUB5t$|Z(5&4U_&}}N$C<{kjrki{%&YE;hH>^dM5d}7@ z6)D9Lb28iZB>oCZ^bzSz-?vrP-k*ta6HD7-@!2hO*3MgX+1b+Jky^B+bTVrgX>1>y zmfwd)xt$n>{oe(1Af+e$B-Sz>rBJf9JVz~_prJ}dy;Xm!=@!eGFH3M@uz#$5D#GrZ z-^%2WZAwlyTko~+)yueFzmn%~qp~=h(&lp5fxwkoYyM6x_M;CL>e*(sB&hRaPEex7 zO0^I^Pn;3;yPd6Ws~h=e9h{_EpF85#!^K*|(FhK)Z0?f7sdw4x%i!h&T{Wp69Tbk? zIzHNxbKd<%t=PN8#*S?d7oD4Xza$(2fI#J52I1}6bJBv#=TrNp6l>7)^^%jlE=5}P zh!cr*d^(iRgOvAl!aj}Q)kkoXK??mimEz4a>uyk%Hv7Uzb{^wOPV!RWw`rZ0Z{{Vz zwLusI-*pwxhd2<(fhWb2sqJ^P-Byv-oVpydb+GapD`51zDz8cqD4vVH&ESeUs3JWj zU7d-+TnWOoiEVfN>2wg;<35C$P*Bw|Hwor-~xKYvo&J>>{m)G+3N2Zg zd$H49iO!#3#s1^bqPjb=ru?GbMJvO4Od*Z8$!>MyqT|Lbu5z7XST=^&q{| z`B-IjTC#mxCUF9kO&OL{73`@XPewXlaA#ZF8kg8WKO2#|vBX4osHzB7_nmr2z-OYd1+>e7})NE)C8%Rj_^mTz_noeNbiW1^^3Va)j_)D1k*aO z%SkIr4)dT?i^?teZTjI#ciy49Stje#; z-zPZ~3AVj6yn@=o6Pp@I6$8`QHZ^e*Xca7H^bGu=p1f{XRA>}Q?bDAcGA(-W+~auw z*d|&^cJg3C*!iI_WB!6>0^?VqC((cq|#MMo&nz@F)KxkuE>tk}1~PD`%P@gq0{{R?lTzZn_wd_+ zlck85?N@oT%=;_OA4mkLxdDRW#)Gzj7Lobx;r~-J9x2IyKOtC>&7I0Q4&Sg#B~23X z*^jg;whpLO=fKYW^+}cMXHb>LcH_WK`Nr zBa__<iL`}Lo%&zpJg zd!RdepM6Mq1$j)=Y@3glF%l}6+eUbj$4t&N(k3_g+)cK>a_($8;0h6+Q+VOVB>0>2 zFyZHf4FmY~gUGv=0~X&4=6=J@4};b#-v@p)?X-*Ts@5wG4vhZORg2t)l4;%5THkjy zy5zO*M76xtf9N%L2-~aQ?yhNqG2Pcw2E$g1(@-0OFglbi(c#0aahh(yK@W{bhTg;X z5zHH|rwXV?q76L)V34#*JWB(WUGq20B+5IYA^a_Mc>n&6uz~mLVGqAyY9y-L15xXG zh?Dd1=a3h?TOzWfJ8tRKMDs}^8o!5iv<~?-utrE*8{$q?M!DwWzI|{pP>Fr>xGnsi z;d7(AT{n5)8^wRW@8<<#CRmw;#j#ZSKdDI>2WBQoH;Gx6khb5!p}@>BkgPc1F(21k zOoSAM^%?uu)DXwi_-->H+%EcJOvwm6CKB+chHC)it>y-e4r32@B3B+(M{WgI$u`;x zK`cHD44fWEc7=4H+bxQ1(MGV_!9|^VCCEdHN-x5)3W2PQBT0Un^{c5gJ=@2nEu}W0H3mO!4AT-p1()+3L%Tt`6ncs_FZ9u zKe#-<{K@L1N$)IVNBx<641J#*Ixn}p&T7`}8x!Q5MD)g*{wxSO;~#q0AU(|UFXTlM zOvYJd|IZ&RD=s1m8ZburwAh?7phOGp;H=YYm(rXn)t!n3Z}z*$*lKKco67sHrI=o* zMlMAT+;Zc>�G{nOVVr`qHUaC<>nico63f=JuFX`R311N8eQrLw${M+ODknB@*wj z35QsixzauaG2j#v$BlJSVhEq||eHi%Vjoa@kkC@_-1kG11H8@JYeS~w#Y_(56 zigTAOo*|&jQ%cQC2TH1Dk52!IS|AEG`;zjNlvmaq!4M&aU?q55O`xSdBAEAzyDj;8 z*3y?Y-yL9AsaadEf?2B5sljCPcgnNSv9ej1lZlCQ7#442V&QRC_#v?B$~A@u(~@}$ zwKg`xh$*YMGm5C5c<1r9i6djM(ptEpPDN2^t;znt86m~RVv6oRla>e!r09+S6R9Y5 zJhX|oapvh{wqh#7GNWvGA3c<5Miqp_)QY9w{h9C%GVPcwF5nM*G@VYy{wSKR0-5k< zd2|6AcynHKvH{nyba>U;9stLi#pPuaW~!5?eI>2$vCRsqc42{1Fnl_>KU*CD~C(dk;)}?UvX;`*?L?hvI z$g_|Z%c)F)$${M7b028OrZxD=7q@pm)?QM6u4WnN3rAz5@^OVjTVKb>x(#;F;z4f= z(zqZ?sy;>1ehEC&@dhQN{duD4^59!p-Jiqv8Y49P z3|d$IycB;*%f@smUoZCKHi<8`n4gS!=CW2Cf+v#xt?<%Rb*|uQfGX0{be#IpceaPT z6>dLpT?@zjnE6IjRIP)JhFT$SXo^EA;k2s8I(2?%-$aYT0K$X90UIZ6L{@eSmlz5_ z1+XkfydCX}K0xT2R=$%wJ^$SU!8{BN&)H2VQ(TJi-z5SM(&zx$RYJOQyp94HRk`u+A59BELFgiGXB!1$7l{Lo1J5%%fv=9k+P!qkHC=m7 zWQ+8lBkUg_!CGw-97f6aua~_>0a_Yx6J3aDcS(1|ajB9}q5g zuF`=l+NqCGGoCczG{7kWA;vZ@-54rwz1gm&E#&J%%rG`WAzXmCcxq{7M-H z1oX_bDmP;ZtYW0!XNfmA@?M&M?f-t!-)=VDFf3O!mzP&?mdC$l-df~_I$Zq24o=T& zt#a>0!k&AMFMaLySP?;=@L?uRnWA~d`8VIRfQ8zDxI#>|YCueMdt(?3rBBiiGHY0&LlpLEHN+IVI9bsio_`pJ5WJw=k@y|@@;Fo6IfNOG*lc! z!q+|U4$ereCJEFfu`Z{}`s}JWiDvGYMZ{l?MSEfv>!}qoAAw$GOCt4BpmJ}z2!hC- z^Q5THvI1HR(E>PV$2h6qV?Es%r*U@?7?h2T6V6r76gq+tx)4mV#3h{(LAWAejApti z*R0xNDm9jzh$3rtcPfgJv;_NvNk&5(KsjLU72i(J*Wm|39=x*x<|J&XfiQI>oZ5u^ zj2@*kSR>e#|K~X%iAp}LK^%IF*%@rW4`*L zs;Tc5K_6y-zKxE zeuO+BbV?}P8ZMKQ%9Uigg)_(};KxDz=+d(L8OqZFIc;+)r^)3EQ*7E3ipgw&SX-6M z&WZh&Ul@|M7CYC3Pk|bIyR+<0R`J&%!-x0WI{2P9aj)~G-SA)7QV`-bi~E*SclG3&S(funcKD|A3zI=QD?bg5N35!9F*>r- zJ|?u&E(A<0x%ssAZ#r_8uKYZ6`x$42e3qY&Xso{tVcRVrdjBt=L=?h%PDGCE^Ox&) z42l5d21O_Y1R0|;6$8#h_2dr8O73qFD%%#&@VD;N-cjEl=adhXJyL!W4s{#z92$>_ z9$Ga>-`cF4&x&yk9VUmab_Y`mYLgD4JV%>xV>?Tr8smuhjXRT1a84m)b*UqXyKeV{ zs}Mmh+I)4vBGwJqFQOFICbre?PnoW63*iicsi>12{kuyQ>Io@2R`$g^pZF~ERZ%Zn zn|XC@z==u%6xeKTi#g;#;tfKipt?isQExXLOaukyv&}&qMqP8k4wu_0lRmE`P`KEM zkM>Xhn7-4<5~6|KpF*=^V|B*j%njy|FO{VppXQICNgO=9Rl91^043M5V9tCH=aR-h zf`#oF^M4QoJE?;KPCU@l)H9vcF%oYEE|T*&U(26FuCr**{;#A$qWTc$>cTf#^c7ww)?pJsd8^7#&$+a$BYq~Vk?{sy9 zQKUKthk&(f#Ag4=8sa#jO|faM(f>-+UCU(9wTOT_fCeIcrm5?(QcWDSR$lTNZ@`u! zcA?KkBivYSa1nkVtk!91c;|~1W+&*~n&|mEuyF33t0(Uuobv04UeK{qQGd}8zWpbW z>cp?U~Y>L|EPYugG$HANdlMgycv49i~Gxj%xwI}UxT<$R)yVtN33_`pjx5`Z5eC3nG=pV!;?+9dg7dj-c`}1vmDg z0u}tUngGFhJXVqV#V%Qc%D~-41$-(?*D}BJY<+UOQY$r)DZki zc%Gy$VK=zUZU-vl@b7Ee;P6Ql9%Rj3usLdysyy`AuwXPMzW>y{_t{XBuUfBJ4)R?5ENuHF zc4pJoHDp`mN5NzNc{+Kk;7^AlLrYo^nI$<7mNA5mB%zV3HdCElQwsKbsqlM&Zft&| zZisk6h*g8Hk@lg``!E+St$)H5$0<~(`=(=OvZtm8R4PH{tOV?~eOi73vNx_#S_2;z&isaA-u1>QtnDyB; z6L|WChT~Sr2W2Dl+f3%ZL^84>YkSp$4?15kiC#ei)t9eH`I0fHQ@hp&H_=EsrU)`- z@MH^1a^|^#ydwjU1MnJOLx-DkSpxjGfduq<;w*U3Ojtf_wILWzIx_iv!!c0a;6DvU zLQw-VomC1VK6^KSl8yk~1_0b>@=SWIV5r#U`>UFj6%s`wNXMjE{1k4>KOVFG|rWvEX zXox;+tu>J&25{UgwUc-B1wrp@?(Y9=_0Ui|{0KXIo^_McIN6ptHrZjpS4l_xDi>vDMYTT%|kU zd*DrqB3#gyAPwj*GV#$$N@5PP<%{$yM6U>A$tnKfSXuy$l(5(B zI=t#Og8Jqd&YVOZzJnG3hbaufd2Vy~;9eEgW2|XKk{-;G@~? zL|=w^NU#?4Ri;c8$joYr%8XsfT%PxIsDsM01rA<@%fi@jhf*B`qxr=qY+E;edY!h< z4fq(F+Ohtk`JKh@_UeHkuC{5|BJ$>_t{&eVPg!ovMiplFH9gaK29tpMuc@(LdP4bY z2d{e;6_xUn#(T~8Df>Ji4;va#Km9^)nB7Q@#tv4B-v(&V^xT=8%hN7vD%@pi(evtq zwno+;?WvBbQl;A&%DI?nG|}zEgxoPgK_di}{6dQ)ro~9w!Vja?q*S1E$JK5K308hX z6$KvTqn0Gp7k`P>&lE!c#nq@7eZyb|ijf}o6*;&kxLR+|Uub%xq*8N4Qvp#l$#%s~ z(YaO+x~Db#|@UmrUI;{iU}Z6d_f-er6UIdW!mMzG^s^J zy5bZO!3*L6Lj}NE59rS0Qm5~+8$ux5Cs@zmqDLu(SFqda=drktt3ujOxIn9{MX8rT z`$}9B@cnEfOr4kj;vLxv<}n*NSTyA-vxm8X$E15p-l6T7s0m6m*i&qALzN=T;)+)T z#jy6-7}*linU`{;ClWN`vnV91TurkB8&h0=0kx~zg{T|}S!&?a_%=}||8+X& z_)Un9%`lmW#lkoE`Xs$h5rC38O-*TS2;3NO_7%OQ`47Vtfaxp1V}8`^{$}(`aJ}#Y z?0J|0*dY0QA!B@nRvC0DgcQvOe3A&jLlp`d!R0s->l4egv=U?g;Q<@2xia|z*&X05 zni@7NkW~L?-grnt@;gcL9`+^{P$FoC|K$ldCU{8P81 zJ{K{QA=-0guFqMZE7$RQwK!yS!1^iJGUV+B|24K7s*A{sO%!RA5dG()fQ29VPF27= zsZ15W*$LS(5W+;rpJlmDx<1C$0c}|Yzo#c0eLts%q>4Fx+v{dKiQ2x|dT_3;KGm7? zK04HOG6ZqDyzm}FUhAwwEPaT0UkJeeGJd1p0ZC$MSl3!SE7kko4)>F^uQPmZPWsxl zO|MWwo3!M?b19pr+maEH{8zY%9@4Wv*A;5sk2qIkJw(Moq5SZx%OA_)ufRR7XEHJ= z_ja#@E{X6{EJd_s;=ONp*R5upBF4~)0=O6On)A5KUpCNkk+CVJLM~l^QE7pu{bgF| ze<1j9rg$9?z6tKk@1$Gij|B9Wl7OI4A3P9yN5t2OgtCLSj(=`P^@-^`d@Y{st~2a4 zvTGn6-LpN6fDOV!#Yj|sL3lX4#-#BQm4OkS&mSEi_h<0xXO!3RAI0YfI{Bkn8%5VW zah_$M=Cj_g?2aF^|99JAVfxfbIw3HS#{J7-Z;+1s{Y~OkJ&2O%Hzzun?+K(kh98FS zCm#e6orLKhNNKR@x&QDViGommLxs-T|Iq-o?Y>U%jlN~OFX}|w2U)?gVHtlE*CND) z<@xsDaqW8S@(Rhv5$Qn-nAXxi5w9D1B%4w_q;zQN0_F1uydKSL^l}|~16NX)w1i83 zY!qe7Wi&z<$z}v7kzSDzrNF8VB;>{Q~*x2dZZmp>FY*2$iT^OwT|wB`uQIf zy@P}d=`cn^@9#}e$0&aVID(unE?A9QlMaJmMM3zY?e2)mL>2tMsUeu74uSZ>agBfK z>~s*gFqZT^`J01j?T5Q>*m1}gZ|%2GkMrb|;;Ja2(osTt>kA9<$v!y9n6tPuURN2ijN8le%2!;-hfA@ZYR3uWb zLBRELac0wbZpKd)CG`3Ccw_xe-Xfz=j-wE=J@hBN7C_ag?a5%xhl>V!48cDcB@G2C zPZ#p{WQ)qLAOBVhm|E2ig-EqX*KMHObyH83pR@3MH(YElNRR-K;UjMs{S#lnhnt0h zA&30kScmkB@c+12*Z`5C-@aVjpV!i64|^mUBT28!Lqt}8jTmZJN}T@F8~^*^zZ3d@ zkok^jNMElk1evhv)poB}3yEg$$AxF=5{a|0F8^|S#j{X%SlrR_ktd5V zakxac)c+q)@9_pc;_BB0>JVe6oN%#T!g&q^5gt+X{nkr2M#v>>A^gTMRqxAXXzY{- z-^@Ce_1*(Nmva03tH)7zKdBwzK~v~jjhbnHb)?whBqMk@V;~_tu5+%zA5U|w&c3!( zC0jL-O%@WFaezF~SU9^VK50;7d4a|hGOm$Je^5}=Z|ZGIs&@)q3tlwitAIs==sr(| zi8u5ysX*~%Uz(SY9Mb(?@slLW&HCvF)qVEshWpRKwtz{8*nU2!w&b2r>&YLWm7B)u zFrnkdDv>X>f<*|b*zLEC{XR|e`zZ#dXPo8(pRTO877=#X|2K_RA?-s#F?n0o@#m&3 z@fxs#;Z*56&ofj0=xqr+zWCqu_2vQlIUr%EV+X|ZbrD5XHdD#vnyr`qdY^ukMEf*F z4C$~%C|oI$lrB+ALk8E*7#as9cdMHAeG;~FpRjFI(;wVBj? z9;i{FJRY^5vV{AEFU5|oNvl{+c0x=Q9~?lih@Mo4?Kj8$h5jupMu+Nab9L2#SA8JF z)fmghxtOB4`d?}^p(%T8^(<*E1p0(9BKYop;3Z~u0KP+94)?Z?4O8O8ZWt=b-=6Q~ zYy1vEiv4ucQG4;s$h?fVVXkxOog>1!Z*e z^#k<3E>u^zD*Mm#&vO#DO$-aQCMrd;!vzM=rTk9tw_4_8y~MgIBng%U0iykxUWR%^ zn@o3yjH00##R3A!tj>5tx@obbsPpE<_R+EX%zUa&Z-s(h@K29b`X!yOn@`tFhlg?7 zXY+9V>~8hp=@GW5(3Iw+Z@K?l3p6kuA1Xry`G6?kJ3B)`9yC1K4?*hhss5GuwYT!F zfB)ZzI|?emuL6S)8Sl&0`A(r5d zc$v^s6j+Fg-5wS^-gw0vfg2E0;qiI z?E$m~f?5jz?()pY1hO4ZDZChY=pKNg>j~VfhN$Q+eqbn4@7ih6FjVw5-n$AnP4mN& z_16baA|Xamsk*Yl`q$McpC13n88Vj}{N{fvO6UkU_kxB}Kx!28eW_Y1=DWpCeO!as z6|k+yGz=9~C+efy=23s8m2h{mx`{1!gUfhp*T($cdIcMM7{&UY%1x!a@r>^>n`g(5 zvTFw30hIw(Vns0;C7z9kTxMe;)`{uI$e-*aBqb$VT*g0Vv061CQN7%CCIL^Vz!AsR zVjBoW2DH>J{ehCPmUQvAqV?*=AMem^O9yGX4|e+1Zs%*6OYj(YsM;ECu`xfaquKXa zJC5c#ct7JddVmQytwKCQJl)v-Ibsb%Fqye_`-eRe-}zZgxx#3w)wL1_jI<`Sn2w^p ztYa!Y{Q!S^v{={TlkKSvzL?lt1gw^*$7gthF?ue$*L63ssk1&&+*K;^u(kQ($+PmK z+*Y##BX~E~5peP;SOV@EfMtMz9|`fCnPsW_!)~z#p0HQJiG>>la|!m=%`taeL4Bge zt1)OCA@+L3x6Yg&%J?=SZ3>csS|DmQEYZ7E&;#W6>UaLNo&P2cap=e2;sypUSl<#rauRF%Pn$mgxW~@$?ezO zI`>nv7zHUd|Md`8(`;wFF@zd^)8}(<$>g-|JAW}?*6rD#R?I$Qm6erkLD$sB2arw4 zEE`3LXqli9`~9_&my>fzH#RPELkkWuggr z2ST#b41I>3oLjY$4nDr z!y$`<4&;o*m#U^Mx*TtrD1B~cfNq^3Rb<{3pU`0Yh{3Gb74qC4Bf;W(<_*7R3;n4i zzyCL-u=+;;L6Ha24gURAr5(EIG1y$QYS7BO;B@mh+scjt&f$2DtOkXc^O?yibi1`; zsX{9oZyo$BTv62iTp=T_U-ScIQby^)FNT1#3H~N4l2h3+hM;{XAvg28HZSkRUZc+oYWvaGq@|P zAA#NPYgrl}m3P&-Rj3&m85bBKbEJyf)pRfS=V?*8tOkF=e*SJ{WNRS(1#c=A0HuTT zm28VeiclWUl)crK|K=R@skTuR8h$LLW)AGT_>C-as=;J?)fZ5+^m0wn+lVur_2(&^ zT@UovthxNFEL6ea@cTt-e585s<#h*K+iy$ZIvX~r%zJpq^dqBzfn&jwZ@Wtd*RHf4`>XriIgs8!!>l~rC*?ym*TUQ9@BzzCG0HLSf3!r)yG zlJV3B$DRSy@jsv3ED?-{1=7t!IDRcLE#~bf+=(RBP%J?2Vj>ijUMa39VikGo`(L}I za*dr2QJ&BTga@fDbn+@IxM^95EK@KCUwJYy}A8Pru@e@CWDrR$soY zBw`Y*Ild{Ta=qp9+qHrSkS3=ZP39cY0{b{*EziEZb6MSp{?zYqX?TiCLquP06c)S> z#!QIGs621&eapcN@auYvf4Hfr}^i}Yv~H-}+* zpFlj;s{;B%g?-Ft%Ucd3$FS-rZYOuj5pbOUB=LLyL$yz74KrZQqB_&bTDA8U9(^(? z|7A`b?TM2ijj5$}%dfXeZ+T*1e*FFYj6GA8ci58_PGaf}LQwM)_qQlWz@F{g_s{kl z(?CLxT=X3yDILT^fGki~kCa+5hb|pngZEkpeth* zO*EYs6V`3EEP0?_owcwy(yB?O2G9Cf=IELJ!lWdPEBbp!C z%y+tLY24k{$gC7SSW9eg&0y7H2p#k^q;JL}j5gyGFT(9+Ug-1bLi=npBC*MZVEqc2 z4AtMu->!2kN)~^5k2PuDFr;KCKb))7REqmED3_DdJzm}M70;fg7#0d1F#{uqBaUvNUp$lviob&@^x?^#sX<>qMxyg*QDpTy_x|pJwC!YBy05zQ+4tnM;lHo_ zwo-3%x3i5D|-xjCvyG_72u{eti2= zrn04Wnbh-(*tQ?wTwog*LBmS$&=>_%Db+xl&1RB}ggS))1!Ieg zEn05CA_`*^)@;2x_H8DE!Fo(bn+a8Ce1fwZd5s-F-~Z=z-wOe~UTtYa9QBXD+r!K= zirrodaKj`T%fG$E@ReD!rixLvvPz?wUWY;YOkX?K zsgBcRKA~5QMs#5sz$j$ofkmr@q3o~CZEZYxr#=|WTKqtb4Pu7}F@|*&s2%Bs3r>up z;C1Z}=b46y21Vok0An!*StOz5`70bIg0?8uk|Jp|ezp(yZesrWDV?oFW`bvDsY6}f9Umn{_&iv*&mBNG=U+GsmBw{7r|*vJ)^Qps5@WS? zi+{g6ny165ty6$c_U;1l`wjBQCNZQ%Z4}k*-ScR5G|!uydz(%*dRQjE@|&i6^+a8QB4apb{8t z-mKcYiSrLnL9D!=(r&b!0lTr)a-m^UG!ovTedEMP5d%I10okb#Q2ior}>$$(!3<4U3`lnp?rK5gS(|RSN2_ga( zGr0^=id88r3h~?$b>Yp4&7@>x{jfL+QL}V*+Ax!tlNhpHMA4-;@brv(0B1iL=PsM4 z&+Xhfw(c8TD;i~aU<>8E&^MjGd**?ipwPoXe7rjp>h~Hl_sPFsCqn4MkEEzL(tBA~ zk7DhBhDk{o^sOxuEdqk;cJ230d9&)3n(GBQx?Pv74_e!GLbt~t1+g6nucy1#>d-2^ z-Gd<=B-2VgVx4B2h$f?`G#MoNY8$m_@pUdHTX5rzhT09VW2HcF=&j20=G!~6ra(HR z$&Wd+8r{|zBM~iARso)98T3Q(2DxgtLP+p_v5#oc{ zcxL)fKNKe230jzYID|XfEQciz?MeHtXQ$J(;;5?y`Yc*0Jk7$!*%Q+v6g(r{A=m*t1`;AG%QZ%B58h1(lk%Af+Axt@vOL2FvDdBUePiCozzcjfwcH~)^It;k6_&0m z@0S;01FT$u<16Vp*;Iede{ERT6;+-QIzM{Z7=C?H0ldzN?{L5Z^~zC?o}%cP7zqrm zDk&mhqj805AJ0tOm^AqYD47M|2eHXzKf{p4_rp9qdE4=CD>|&L2O^ z7jgh)NJ58WT-^r!#z!sscANpSIRfU9M630`9HKf3-w$d;Kk@#EE{&0tkNt#c$6sIX zxJBa~9)OR`oiR}6-1%i+D-`&%Ryrtmh){$w%$A$3MD;Q zI|bH$fSv9^9KXeuUKX#jn*sd}g$Y(9FEr`I;D?LNvMw0sRBNTAI?}E&W#3CtJT5f~yOWn?*Jglo zGjg9>wG%5PqUifaO_%EtUkk^0C~Ay6f`3`Qwu@}nFU=w*&`lyg-`*cXzW!2NP?Y^+ zUfy2!=_G}ck`mvpmNGP*%GSqfn5M;UX=1ZHHd4x%OwuZZYUkW|usXAJ}}fzO8c&L4}s##x%FrZzrx+g2>gwn9-;2djbqkF2+fYODRC zevtsd-5pAS;_hB3E~U7;Q=~v}2wJ?jOR-Yi9fDKb-QC^w z{;_SLcg^b^x378jhtFNK#Ia1kB_vXNsLP^Dgjs6i!`n6rP8pfAS7!Pz3!|6}^H;>1 zBpUX9d5vEgX-hP3*Z6dl)E18u{=NqsxdsIVb_`@K44x%=5%aPfkMma9jYN}vc-DdS z^BLi9KmS|67ywt)PU&#roF9KW2f+l?454sbBD)e;e=TUn*$$^<*8Nxq&FMTA#$u;P zK@sX6e*RF0flg#4aDrFMGPEO$_c2CLb9axwdxv~!gHqy1y014Y7n2g@|FQrajY-2( zI=&JD$<3oES?{Z3?Bijk%52mdWj6M1hmAyzMg##c<)BO=Uws6pkgcpiQ~-v_a;rzD z-N2z3NrO=dMldP&cWvo##;z7AL>54TkULnX}FKgwCSefEPBoyKe|M?Q=;nT**8Lx*E}fQ_E{8!+q-n!(=~U-<{q?31s7!#Jb1&O)^_)6$ap1%8Lb>WATUzkcLF*S$`|l zDmk$L3y4n79j^Wwl6QxON3__%=v0Rzo1$rJ0SkgBA!1&!Q>5=lTCoBnHW{g;rk*uB zlw-QvrmIS6oTEhw z^fhcaCPvRq^#X%Ph;sHS!TTwv9YNXR{Yn6uECAOze#5u*6;XH5@!m z&YT`iGN?=Y@i|NO z8tR~0wgHzxbF7pyH!4!jZ{~OK=H1Ssub$=Qdg*A}M2(UP`o~|~q zI^|%~1?I6b7R~# z%NAOggA|eOReTWsBz3aCV6oJhd+!XRTr0gJCofZR30ld0i0~{HRUGCCQ8l@tX237# zqeh)=LN;oi_*x~x*}OsWtVor}_CkyAVGuoc z&2vq%AC!FnV8#m;)+RV9;1BlL+0U8IC4$U$6F&E1p}B8cT{#i3KkJLYo}dOq#)x%a z??3gY6HA+NqWK6cx5EW7JH zmc)w`W2oBI2z$lyWdRNc=t=hDXF%8n;Cd#{Oo)=wOoHEbtt^e@TNLu!V$@F;^^g7* zL5K$1*Hg6~l!Qftg1nY9vZM@zQBERLqP;{4t3Pa5s2@Zzkj){5b>C*HrsC`x;gA;%?56z@VsUC0sfTr1RL->WfmC35`W_pE zp9zO?$Mn6js`6ooUW>}dxE|f_c{Jf=)1d4e2?LQ;;hU6!_Lrac0S$*KT46Qe?ZXc) zt}X_~PQoSLX}=}xqjBeK?ZM2{yDBYm(8GI#S}MQA4@`EJx@xAV{z8r!mG&YFjd$& zeTz|WX;#bda75%x(z;s3U0_ocmYbbvx<Jifd>w1$Jn#Gq`G>iR4uz@v71{&x5FLd{u!6nJ7nH3iy3+f6sAmi3$e+!(4!Io~0hC~%`V}wbC zhcywG8)iW(#)!O0hkB%z7J{}j!*nHzSJl1p@36HFAU3T`!*=?@WfQF z>PLtkKvu5J+%a3vFi= zbE*T$;atGwQ6D7mRft8VFgwGE$|mDyCyYi7FW+(SpKsX(-ENv;k6^!nd^Q47$(%{( zgBrDn{--{m!R>~;r?48FJ(ve(N-cW-(J<2u#Th_}n*le04k|B6pH8P>FH>PrsB zE2!mJW*Zi;ne6)zsWxVqWTv2H-Xg^O93rywEJOOU-R+YWLvk>jKZO6-U^Z-ItMfKN zh3$A9I}}bPJ;)6ktSP&AU7I6fjw$*OAab)2G**U7g(-@h#P=;Ny{4C!30pRPrRDBq z&8`!4)95}fa=$iJ6`yoksa6x>t5!@4c{<~$`op3w-IKH?>i zbQ5O#KtxGtCHCG&>(~&ou>r5Z>V^@Yk$;6=uxr%d6M_W!FR~(VxKWTxMKc=O)m>39 zKU=KTLn0iz&J}|0x9M_81~Ad1u|^pA<9I_?oD*=dYmD<2sUr#f-T`RI_#6;?q|pg* z2&%)snxyPQ)IEu4YewHqFm=G4xWW5;^?qf*wVBKIs* z<_G$7)m$z~;(ec)2m}gP%c8L`il?-zsuY)D#jmFEb*CHznILcN97q?0Phlw>?)hD)msar1KpXS zb(irOagTiA5B(%uvH*~z!J>5m+ z1o0Wq%V#zJMbWH2VBu0|&2nEXDsS`RAylfx#uik68UW;T%XKEF<6{6Ij={oyY~>w3 z&6`zHUa%Uv;`U4qp4f{v&^AboKO^?gRWPVK9w zD?pJlEnZUC96;ASPW)6ruKcC*Bp&s)SL?4R+(%nymzq}G?1<$IEwcZfG)yu$_L_&6 zZJ9a=8v{M=?`LpG{PitcT_8~-R*_3*XZP*gWBbqx5>{b@;A8BE6xk{^qxP6>3p~+z z1}WV0HEL;m=kNL*omZl2WxSFBL%%x)9NvgO;IXo?pS+q-^7;o`h9hSFsmx+1*f{;CV7TaE+Ra;8svE_Qajg z{K9LyKqlKTrMBY_E)%BvUj+GHm$%3L#o=+$&9fc|wjlcI2atb4B6tM(mK=lr}7gl8NhIx7=#Z#uh9FC`TfOyp_dn)iRQ?FxO_J zjmpp$@`2*pss%obR;{hY{EM;VSw$2zdtnpWW)PINCrpoH-I&fDbszN&!1FP3(SDBm z-?O%+mFfmzn|_*RIK0hA4|2sxAn6SyER@41y%!*UHM76B!|=d9?`uQsu~;+9qCh}+ zHI~3qV%0c?f%+5|mm5TcyCsbp6kx5qlg({Y^OY{9DsbO`O3B1xe7QqXw=($2xdNGX z89BM=$1KwJss^#{+N)Er#vko9uKb&$$<^mE$|a)vT@WT9R>l6LLK#w%wL)tg_c(?C zD@t`dHgUgJZ|k;IST4K~%>R%NM?x!o8R@vpJz4&l661N0RaJLD8OP;MSbZ0n6i6UH z7&+Igbb%{4!CDV;gtmUNT zq!0~g8Pi-c@z5smYf7 zA0shiAM-Bp2S_KeOo7~MOSVCn`gR5rtMyI_?kZhW{V7r^V*|i}!O=^C;^p*=iQ*2^ zc$3v#+^Q^5b;)=tjI#(!xd~L3aG{{P#eOj@g!jK?jRY6pr z)Mv}9_!8?_q21&A~!ehM!*2fo4Bv&9LWzM#~Vgv%Fz;cf0vuGJpz%@AkVrm#Qe z{%Z=E7R}PaVi#EerZl|9MrrOG=Qoo>B6~q}PdJxwzjf3q9fNP-K3OlSC~XGe_tTk` zXmUQovpM3LIW4D!tr*4S|2Cojt$+N)fU$$MT0T(01Zt7X`B|2rcYg3YDzT>F2HMYZ z&gK3CpRf}XmSTdaPUBvoflNa zX|g97FyF*UWtB6sQD@QonBb z5UfycZEa^lPN#e^hT-~05ZL?3Y0ykD>B-xLNyL^mE@UzE3*IEzZWDZ-(`fWzYx7(U z;;T&i__sTCPc1Af-TC=fxG^7e%vvfbi*bkX6T`899CS??996tTVyBY)J+~6ikT^F< z$ftB3I8o}hUh-xFbqixM5aBQ?E7{5yR8%Q245I6ATOU!z$P9AWjHX|gF;!oWs|jrzyM{Mgqe;KJL^v`XQJ)S@VmV5uZ1q#Zt{OveDJ5rvX$ z?u|Bt_aXAN=j}^1{55&;hKrLzgHs3-LqR6wHrY3b4!2R`0I9-&%vb>jyCLl_{~27T z3HksHhx+1`EPm`U4hnpQ)B07WN1k*d>~}2o*jwC}!90KGYUc`lZF;Z75vrC9O zqqIA!M?xW{9Xq3&NOjl+>^@i(NpmlrF0xgRWjs&aJb}hr+sCJEdllbBkCn+w+RJT? zUbCQP$rw!std7g*JjBQwXaCcNIL(j2ScvgA#GAr{fQBR*Ra=rzHI7?=#?YwpKGP)>7~$g{WyTMr8UCaS4@XA}nL~jpV@`MHf{nsjFCU6cZUH7-v5Wup5NNih=U#ESYx+C&ayf zgPJ;7%qiFjzuCv5Cy{iN5KJ}1^&B^$4%pDEK_bn2>1R3Z5WG=irz^vLgJ?RBa zkVS!!e2+w5S=hRgW?{;R-yP?WZb9qc(nNiuZItDmLw|C6C&D&7s7!xkX;Mf&qfHgB zE5QU0znDr~ntm!NYI`qZT-tBOmAj1<$SZ$Qu+XUQp?s+N=;UkWn>a(Jmh7)xHuR|~ znWs-Xkj+lu<468oqOLd|N%nG&V2(qvjlq?w{#ZM{PWVYSs3Oor1?lL6i{Qa2wRY9P zoO}>_y(cV-y3^h1Yx%<1^|m2G^wlLk0luHkCOZPFNBVPMq)XBg$u8 zRN4=~fpf(kY~Mpg3T=HcYtQLOLBhz+kt!muW%Rkm@3r3-BR4SaSita=lk~wPtV3#) zQNkAP!w5txQk?itU_FPTyYDk4zHfLSr z)xX2Sl*!E!656V*-~K{7@m?1b5}P`V>)-e)dI$}jvhQPCN@yWGu&dzg+!5Mr>`3O< zLY73a4O?S0(GLQV1KTASiMl}1HdF%`t||H>%cuz#fTSNhJWh)^R}+z#oQ&$&W27J7 z6a9a$_BSmm7(?-ghe{MG7+h0WlK^D2IKzOK?kFm1$RQR`pYl=mrw6p3X^J{5wkn^2^7WI8M=;}%=dl$K_f2mi%>SThMFeYN9Nsgj4Mu<&}}MK zHIg8aF)^v!NOL4z%-S~@rnDJ zy8(h4Zz3WDr6?YMPQd4%`pvAB3}TZ#Q{ZU#lXg>;RT+D?Np}ioME6r9ZaX>^Azsq^ z#OhO03wL*LG&m^-AYHHlF#+&L0gIUXu+$8hOu9QLIU(#JkR!{tOh6&HvxOy));jhU zt*fZRX*Jjl!H!1z2emNGKL1yKjDcPvPfXB%-M}|CahSI5pUWcNKW4kYR-Tq!Y`bsG zy2+NQ4`6Q&h(D}y`v&busHlpC3k0cV3a5vH63X5%{%k)U?Ept0s`P z2fgXwVi7HfxVr;lHCTeey6=N}nLg#-b$M)j4#vJnh5zXC;<=63Nc+u+gVh5`qkTix z59z`x(;N;p(ce(ppU6yNSiY;drx0HvhN*XwI{PMLQtUs4^g;|C*6H+~CIAmnEnM;g z2HJ4&py-QI9+u7SI@1MR4J%OG+_O%`5kI%O%J?BoOXXlqy~+XR+rxx@D7DkZ|C}= zef#{dyt^xFeg2~TzzX^bCZvKqg;nqBlLKNW-1alf%*?7Pd&axv1FoApn%{$7Q|}dH z)mC=&rf5siOSL+&5RKzv*YnpPd}moez8$S%Kf+Kx6z|H7cFE;?N3zmd<22*80p>rO z-^(~Ax_#Iwj*Ue@YT#V)O_V1bnDDo-+lQVuvCZ}it}Ej8p65-go$Ky1Y0=pKGdZw~ z6>u-80lQNNo=`!shb?)q{D;dCyw2=-qHJz}@JocG@_Q-QOpC9cfbL5JXNU+JsB+`n zj25{A`V}vRRr(Ll%R_i~!vFnqDurXvkfvdRJL2xTDyxus!sN&kyPf;)?67+jKY}|7 zi?-G9K9}DvhP;vw!aDvch4v<=Wd15FpKl@s^P%mw_hRbY+Zdi+vli|N-0eswv*4Wd zVEL|B;^xK}?~g^Ko4t{Yi}|aIk7AU-i8Ino2Lhk z6Z6{x|5AHOjaN5cFRquf!*;vJb47R{@7Slo)M>(Y8wjv)qJ`d9Nu!RDU|J|A|K(Qot%Cl|;I8A^W|=(%)>qnxf9>kJlST#CAj9 zSHYf5XWxu4ot(w}Rc^_LLe8_sgv5$!5cXfcJyDFk{+k;pnYs2&Mt&x1kn69dRDbXB z%Q>@R@7zc~w`@b$1hu>hBwf9ar3)u{p)$ei6HQ~jJ~akS$TcE~M>1*&?Q3BHfL5}+ zI5@WFGcLGkxRzv;y#7l{jreK-mqFHOY@%r} zImHd*`Cg;X1O?&wRV<3Y_n^9{%(L>=H{loDL^1aZBT$aYUVPg%gbd_* zKp9wIg|c_^^AY{{+W9E${{7+O)h2$MltAo)WHb<%W;{sQw&2$9fXY!5)+| zSY=6Hl)O(;y`i$b>%jpbXvl+4=%fynysgmw8KkdQ)|G}QY{_{6R~FwA8p>DxxUbms zI36Y39?LrR<~vMtf#+HF~R!O7Q4fJ+6c40f|7%C<;!FoDf5hV zGo5Aw%3|nk^1DMFSlvoPwe7l8WQN+!U!}!zf1bCxpWmtv2>c^JOzBhCERLMUjN>N_ z_Dr31?FBhu}Lx`*@)K0BV30ph_4t_*b*!hyL!SK4(mH2Cp5fbd?(iZswy}xjcRt z0=rdaCPjyLY}^lqKY!Rrs!_FOxyRVY-<+dAzvQ0R)%#6Z}tP)C~y#|v;jcH#{4<+$H(P+l&H$!DK_k}N-B(D_ztNAs6Rh0@Z zRJl&a6$Y)De`?oJTlrhTRt!GDqAzd*!4H3`Je2P#9_nkJBs=-TF<&(Fi`}*UYRf&O zzx`|giiMpDZ8Q3O*3~-zJnVgb66@gg>K- zq8`6_h++$Gpc*%!+}*C1u=&uFS2lYNgbWOZ6+ejRSB`8}WVN0I95ERg9?SU5S{8={PWVR5zUD?a zKkQPEA`5H1wV0JgXo)+OscpU|wqLQB%HQ68oO8`z z)LkTb`zYD2se}^-^(4|l!@BB`I7NE?h7(FvSE)b1w%tp$|EDZ?P&~a`62eZctqS$+ zBwhH!#J_99$VNRE8XV2lNAlRr3=Sp5PKFAaLqm?M(`qe`0W12j`r(F(F8?vt4KjE; z#KjRa1KL&6LwW4xm%!5hVh+rDxx2Z8K1jUr*v?fsW(n!7w$%I9@Y=6fDO4O#O$cO%ASQBO)C9>yR<0DJHsBq?BWl zwpe;nCfZ)+m?iiongHw$jw>uGUaqlV49umGtTM>xw7Pfvq_=#GLP(hNcL|!tZMXbE zG0Roc2-$Rb_D(B)p)gQAD-19|Lvk%3C_-c|21Qk{*}z&XH8B_HMel9 zVMXixJ6r0o6h$l=22279`+D6@?N#pihJb24iIIA}=1v5X^B=>IVT%yhhB)YVhf`_| zMZ6ZENiN^s6`6|FPl;P|qAL8dRvTl11Afmhk<~PR@yop%9W|A^>3M$F3qRaS=(W0M zu_zkaBWW0^+2G4tN8cT;Dgb;UmC`wRI>*4i;cqejwH?nX;5G_*j^~yke*4zih8jM@|?e7MAF`3qc3^uZXcjm|YgE;N#@oQr;_))GK22k?5av<9p2Hr-7IbCPLHHIlw zLVWy5jRI?ffa~6pRlaB=$?RY270L9sZpu2NuZi`q9d>jgi|)dkG~}MvTT*n61GVt)LML>mF<@5stsHwLwL>hm@j#@ z%e?CUZZ&rtmcX&WtAtFbyrUSdeB$@*61lTAUIbV+uDeHq%3tqxru`cQ>us~EI>5L7 zgq?@?B@~}Qy1%he5em9OyJq+MkC6ma&CHGkOvS?wrV4AJjXkBg1}H}bBD*{LSF=8b zTu{5#QmNcevJLC=Z?}_$u8O7?&er3YW#$S;1CGWn%dks_8HRs|>puAeT2?w=hlXrb zSNIm|=k-}WICl^ok;f3JATkHQeEmLKQ&?|5*-f~MaYW#9G@0LeE3|KKL_|epb<|fd zUrG7>bh)t>rDlj(uWf=vp=!&+)Ku#BKS=1(XGO>Ft-D2}r!96BGm^7#IKsRRi?&K6 zNEW2L_QvQW9{}$pAGIZeo@xZY)#FRvXw)B!C5z8AyCNZ@X4E^&FM0W9&lWf0IxtHB zQE&D?6QX*ZhAOSmZ^(Ay0Mz@T-il`4hq;XMr2fbaq?G^(NuM&e34YxJKh)7KUI*Zj z39CVC;hkgqA!6&s%XPWtsqWg3q`w;aAX1;<(F%M?(6-U_o>I9%YlCPpqv?x%OCv$`V*!WRBHrcS{>9@^En8ENEF*R$HYkHFRRC)Tum9EO4}4UR>;asZ zV-x~JW$`eEs$vt<$ARq6o)E)!xraUd%?f9zt{tZFcSuxh z>IinYKxwKny{NzXR)_Gu$ZTa}b)=pW>JfX*XkXUYPJ}ED#U5JLoA;Y)5PY=i_B%i zM_Dw~w@aY$qu5q&X6Lfu-D?jkcqS`2IV`8W+V` zt1Sa!fY#g`TXo-DzdIV=(7S6t(4|PKSm45dYg>luWLq=p@U`5k)zKRts|IQt5!<_U z?31t0KzdYw$p*^!buOiJZ|w=Liu$@^>n(xZYD=NF_k>kN`Lc&jP%vNs8zj8p1~(3p zl|&iROufi;nDi&q2fvaoj>E7*Xb+ZQIi4A$nZEtm@P(lS_|CN*OW*mjubDk(`r_QB zwe)aYd7hvgIj1OZHbrzZ71n>;RKRjPsGQHcjdCHi7(@@MsD*`plABmMJC1GXzEn*X zKwCoDQ0PqcSoy>_5Mqjiz^zH{bam-_MI)9f7E+drd(T1yQsWpsK22w0o8Pjw&`C?X z8|w^hK}Oriq(znT=jeui%CS41tE^2rOcR|{yvuBMb!g%?tEc;w2YWo3pJ8{&*0Dq5 zeOu2*zM1q3%XKZ_O)^=-{-nv77s~R=DR^?O#?fZ2uYI~p-m{ISqi4HB4hdvEF(scc z%C5~#{eguN8R%WU{O7z>b9ts@x=gd2WU#rTtG%titIykh&tuJ3MY7+L@4Hr|p+@i( z8TEuCYd}>Vh~Yv|9BUiV%Y7f%<;Y#i4e#UMm{jW4%=K%<3 zi+hvoAsIbM|0^;5v)pV)KjKHn=)Q1mG`ywTuFGRZmdu>~9pC%Depi9u-;ig9#`LRi zH?5HQE(?oCssj!M267I>Cvco8LQdR9UHQjv#k8-=K3#`GpQ{v-K5d?S8PH(oeDQ3X^da1tz?`5wRh;rM12e*GJ_vECMyci_ z5&UT>%I92;Bgs-7iIK0B*&<|q?+^RK$??f1`#?%V*W0AIYF{Ls1$x1^qMW+ui+af) z4}P1M+Q`gMOHK`@*z3rrzuDAZVR&t3wBO@bOhbeSsw`J@YAijD77Pi_k2EFgT&wr~ zdaT#_C~BuR43p-jC))Muo#5eW8$|TSElOb5TV4}b1shYi_AO{(3xBeCQ$0g(x%-41 zHtXv*_s^8ls8dwkX{8TT;*kQNE+cxrBgcwf-MWSm>CuI!4Zk?DlR`7R8bVWmX2I1D zZ>9&r4oH)9p{JM<`2l7~z8ad`>ZG?pSj>P-l|wG(rzH(B6 z&~kqOG;gqnDUWOF0Coo=7aImcHM}}Vq`VFd@xLdNn6&B)bf+guNNPYy$}AR<1S342 zzxhuiGr#wLUOwL0i-|KdH2i2c=TP-^>WWq|`D>1GjI(-J&rj^r*Dz+-6(+5UwrPn( z=L?Z0`)wtsZ@oVf_6l`phrl|Js+u)tST;iS{&HkH*UQ{+?9L!J)w8LqG6q^E_%`A? za?n@GA*Ca+Pt@aWJXoz8Vb*IoK{(C(J!aoiyTK^s?}S@f z!ErXXWa0y~(LYh~`@YmR>xm3Xc3V93!z3IOH^E^*0J4ey0!x~|_AB97D>PdnxPtA( za+2y1|0Fn9CPn&sGMcQGlacK_t2q2M`+~r?WN}&moY$E1*~W(-A(MIbNb_s1$!<@m z`SEK`>$`spdw0lynlv&#gY%z^OPVrl<$z~lIJ7nT6YqW(*>_300Idu77gt?4ATH`w zi9v0P9)80=vb(v~_o;&KS4xP^p4+Q=jzdRFg{AdLnkFL$YVI;QhOlOg0c9g2&NQ$% zsRI6iuP?>-y2qO3=9G%6>b9jUeXDtkz))Ksosw znZnxWwp7-*>RuXL%**TF1((qY3el4k7fl87S#Q<=7u#k6Tk(OPl;8%maG?otovL6- zG4B@@mBZRjwvo-8=!q>x3ZR_s2z|?p5d9#t?U7}W0|6%hTljgzm4K4+yOIS$cx1U9 z(^sxCaGid1Jjo(da?ux$h>Wh^QzEgs{(jd-%+`f5jJNkTzwR}!WMvKi*~^zz0`B%` z4%a!VadZ2lZ7srQhb$~=B51P2o*8b9&f#>Uq2x&7CvEPRQN*h-aPhR!YjB+7YJ(Io zw@l9zI#7^C$7|29S)DvvZp7q7o}1bETrLt$@ISrtoCxk@GLHCXQsj6>rKnX>-^D3b z50BZN;qb@7 zjq9JfjruiS^q4$kg{lSPGD?}$KION zrsA*Q9L-dmk_=|9=p(b^81_OilIaj5d0sm%z}K0uuwj8lld9b-zrzn_s`Iz`^riO4 zGORXo=E`*!vKlcPPEPE-uZ_-!1m{q?H^Eb)&Ap^|Zbj7U;utI8D5!7?TN_+f#(hJ>2Cioj)lN51(nWcmu4 zWUz?9MKlyf=>=(6aY+z)Y}_{|Zs|tYjT<%@3+OUq1O+oL&7?QsuwLaA>*F9G>5q*6$3{Y<|?53Mo33o zt8>jBxL#6BI|vda;`UmvMS6?P6TqQJiqE+Ixu&)&D7Y~c5=vWBq-H2i{ic+@VsxI&MBYE|? zQFi2Vop<0&BKq|kM%ZhJ2Hw%L za})ucsa(Y=Zz$j`2M828C7;HO&<(xln}%!`ITuk0JpKvA%C$0A;lw_}N=vQYC!-#3 zeHmNQ!>xr__YaMI@U&&@9{M-2BI2al3h#Zc4jqrgVP&pYi{?=`o-~m}b1ZYN$)8IH>-1iV-Mk>q8M?9B^-VO{H zw0<6uyNb7{(0IP%K*&da4!8VUMj$jeAMxoyKO;*5FDV3B$z-}H+Na0D(sBrnXWRfS zo!4$F98F0{Utff&)Y%-!SpLgcLroV&zkF`cOG6^PYd`zF`1O)ag@`0<=+!N^+_pkx-No(_1UWog1pw*LBdJC|lHqbPnw{3z-s1Ue-)okKO8u0*SU^60Ds z8b`lT5f@%UBFhsmRHuuSM5qx#xDNgbeHOjFiM4JK&ONImtKq!?R4~Eo9eTg zV&j+l1Ezuy|g^ zR|Vn1SgGL)$1WCKDS>Y1Ijuli<*NshR1WUPOS5LgwI6b~LjB%6;mh;%YfL`Ehki{a z9wj^mD4Z@#d1+b*n>!WiA_6eE{kdNx=XSa{f}X~+6H+f|?}KBoZTPdF?rBGaCT zTZ;07y7%>EOjC!?Bw=^8AFYurNrx5WF6EkLsf|q>f-8-caC78*aId57upEAi>v#IK z2MJa^(qL0=(&VXv9_{n<03ixxsbqpi7+jw;aCQ92UKr zi(t1Yq_;_=ej^RtM5U=ZaeJl+aTjvU9Gw-uRZzQ#(|Y?9$%@InoJ+eB2l2{4S!h2T z`hXQ@;5jyp^+Hu3foZz$hfz4)svsa{u@iY$%wJR-gsl;SRteZdyZREgIT^>3@L*%7 zmjnqiD#U(4rEUsYvIK9c+4HM6by5Ij0#gQ-gSOZsV$O17W7hmTS>p|r6O8j-c^lTX zN`Z}ukH?x{#_P?RcflypP6h^%Z!6XB?CD1*wF{oBhy`w7xpwlxkH!_j1!(@4b&4S< zn_G3h1%X-P!Ym(=1hJC4rs&}Qe`;GcboL2pej_L+JKF?P9J=Y|dk_r?ak=;sFXF5?H$T zv(lmCRCn!_wU1}&ccDK@>Q&vU2`Iv%8`0&6KI^92!psj>7{5xypKCTCwOcCqNmTa3 zd#8GRF{tw`sQq$pIa?06*9E`R2f4?(M%;E8Xvg$2I``j@7^uuLkA~vfqFJ=7CXzTt z(|HNh>ts%OtNxnuLTMcfS2I6~8JiBMtc{(uTmAC&G;%GPa^nsFsN8c=zk0;n0z)pR zkPf6xh|ohZd=wj4gA3bTlv*Bsqv|8`_&(i;$ElLf^hkSl^b`z)8|JI(B_-IU82#uN zaNmG764$zY8aWS091X|YpOWo-4w;5ZpQYzzS>d_lL@L{^{-P7Rq!{@3+=`0z?#(9P z|B-C2xibAj8_2Tkca@N{0Ie!MSF}r!&owQ+Tw?lHB~AUQH57fgFngZ;p7ZT}W?Z-k z-GZuH?f%TbAP#h0?s2ka^fq?pbAa6_%7eEZaceXABApFDDxDdw>~Oq)*=oFd789Y? zO+B`oU?ESlF+_ZSRuLn26nlL9esivt!G6hPL@5_dviRNaHB}EeU>0hmw(ETASzjqf z8pghH`U_~!IP-&0JiZ#?Kdj58UoY|cbpu*YvvZc=ZLhagLhVE}uNKlpGUl5?NkD8m zHNB4Q&n8_PjyAQ${IS#e-ABij9j5s*(aw1$Izuj9x1xmRsx)xkd#L?5d}6_6YWZS~ z$CZ@!_kSq#d!4etp7%`tfdDY+F>JY*9pxsRmKCVlW6gU!bOl;*y$y6W;g>Z(fI%E1 z5TEwoa45{+_rhw1ujpkzl`?M1*`uqK4Vi+$g$-S5agl}2aq_57+i*(pLQRrQ$64+( z**}PWl{KGj`}^&omCtC~Ym6|{3}JACMP!gbkKp4aea_P97A4}kSE{I@@6$2lChwbG;z--O^nG11F{o?8dc zgWN=mQRbv$BRgamV2?KF%y);Qb()UF@;ZO8+r{ zuI|snfL2lgOjnCpnETztIK|zL)fZ3#1+qycM!_MWz<4rwF7WM2DL6mkD&F_PC?Y5%GTU zKWH?1>H;-^Vx^I2DOp9sfi^JJ7$LVcYSjJ0`D7z*kh;?ND(uSY3o$%Ky|0ccx{22F zhzbyi>5w^K3FQuaO^+Ll9XhMla>2iDjnWD%`p=YI#XsvvrM~!VI!ucZcsdu>q4Ruz zj>6Az0rU1GefvHOo9=+pta+5zKagG{BZ6s2I1#a8YA@FISMkey{*XMUS#8jc`o2yQ zE}?7&ERzv10lwC8TKD`B8#xKo8vy9k#1|mIC6g`x*6P%kb~c~$E6c3fg{AMEqTHEyQ%*m`SaK4?s#6C^ zh=VvW8#IhWpyb@2u8i%;0}nEO(RCG6+ZE_FTJQPxaB`VnL|4dmyAfr^Dx3zmdN;#J zew5|@%hq|L()#!jk3(Aj&r)6uxjAz8D5Czjx3zYoY^%_ZgRrwRNjRKDo$vpWf{p{Y z+w~ucz9;4gDl*hz7I9yJ(>BL!yYBu=jcznAdr~hNy|C8>fw4|zu-vTI`&q=< zIiS9e-XxWo*o{8#i2{P3mt!Ds3O0XwqwR4Nrr$H9AtY6RS@we&BV8zPmo{oGd|b>! z26et+-q72*zfa@;vH(_Dl$$nQUM|J-DwQ3|oUb)&z9n94#;H}Sw-!B7OnfC`CE5e% z>|HL08AcNcO+L9D=W2(e_+z# z(%mq16cbM=JW*J1sck!E{meRE7KP55{Zpr2?TDQCa(&OJuTKeInDSN})I@|<;q%(I z`T7BN(T$`(7BR?8F293|c22_2Ve4Fe3kmbyP&W1|ZWYWCSQu?+fj78PC*go&p6yiC zN90CXJsII{3=;&o7WZ+}Z;;uUe@v|pN-7rJDkEgl_m1{d!=#zstn^DQf-$C#dbZ@n_vdi)5YC)+sor#>Ho}mPsYul12qQEuLZudi}b3 zb6}?*vCre{ri_yQ=z8H?JA;B=`U9lxTB*o|&c&@;O=!&sAbRt=YTeV~{T!iU zNcoFKISuyfQx%Z3(KZ|n%~)$r*vW{?rzX3Y)^BTt{I*1Pg(0L&t?Y@6UcWqr^Ta3l zN>dYJAm`qIe6QI_1*QHlU9{U2^-HaIq%rL>y;#-SKA9{E6kqcn703YH5rV{`E#6US zRMKcM2lJ|*N6U8;d!IUrNcF-497^9sNas%*r02Ph(#qG~;C|ACoh?T(NDj(R;p1`E zG1szOCGQ#Uf`S)la)RI$R3j5PQiN>MVnMUhHT!SJblX)B_y5+VlA9aKj!AeZNg}_s z117~W-N*aX)(Zdf8dAW^*7qWT#Q--l{AMOU%sFV*{Svqj8}Oq~$PE!*1O)=`*1P>p zN292y1sD0Z9XzTLg)_1jZ{65Xi;8-3@yAEN%`iDc;xE2FP~>hCCSkE$MfD}_rO<6B z!*D-z$!GsfW|at+NP&ViOIS6FPHKOIl)2{T3i%^mn*@_O-&(9DP0PQrsN-?l&%GX$LPZ;-lJkM+2ZP& zBY{34b2p8ASo66>ykH6>gZ#X2j3_4w#Ig4i7E6WL2qX%aJZ7!|fetVSvMsF;Hmw~o zQT*#;SW2G z<}yXSR*n*eLX^=t?Afh5#q5L5KfXqs0Ki{42lB;wz8PlV@Z#|Qn%MI2!{}<;6KxMlaBXH5B~Lx1!yM!70!5d~#zB4)M**qm_L^4Jb^Ot0y*wj`bcj56 zS>}mdWL0V*pmXhjOsSBY^l5x=qy9}=tnO?oehILj=PBqFWvVVvbIeG6BR13 zBc;XuN&42#`ib6%wE_EtbHYm#g6{*7Ravjw?~e_rlEw9?w$OXq>3LnJ%y^c#nx7dx z?Nb_%7|_0{Go8R6pu=L%;$VUYRTrMw`c1TyT;JD4=9$__p3_@=ZY~!-ar#rJnp6Ye zj&jhN3o)0&IPIC(a}O6>nHf$g305B>k_j_=Hs;l*&f{Vq9Kcr7 zn+~o)@sRb%NVD?>*R!mdas;hWYs^ca{VLGDCE)@6$TWcpRtE!hXO#ZH3C(VjnaO77 z-}XxCZdR&b)u2)PwZ<&aJ7~2qD6hSAJMex2?=cVzl8b2Ov)U!Qfe=H0j{6K)Y~VVN zn{9{YafU4zyh)l4^1j9T6rF;Mg ze~9_AIvRy`fmYpAp1Bm7Vu)TMS!Z^*1qkIT{!YX~$=e#OQnH z9Gjr^TDx2}=K3Z(hrpN~5sDzoCHdDH+>*8i<(0fr%XN0865v6fV;BY!3{Ua*Z&NEa%pEr&uw&)(`L6eg8Y|+5+12Bk)i6Jk)Y*EVqO6%; z1pbze0Hqr(xFTfBu7)xVJi#gh4Zu%FQ@p`TUEz6;Z8}6~Mr-qeQ}Q@`Q{H4fnzoQq$Xe zCr+k&>KqdWzAXm=bBVD#zx{+z2}{Sa{j%4GiG3E(6nz}p{dIC))^Sz>wz^Yx7*yP%Kex% z_@v6vA8^F9`wB&(>z>Ot)t6q~HNiVV=`xYw@o)yuJl3nHhuik|WU90PrrSL#rJ@#>%gSN97zi`m>#)e}5ct->5A2bqbheu>L5p*K^L| z!6!a8QUTG2LH&(Yl<7-$bhfpg4wsXl|JM&rn-!e)zP;+fL)ugfRpz%?Ae58?-dy@q z4o(<$J1!;ErtOiZF9csFzd)>*RP#xtP%eZAl4{lU;^9lV<3f0r>;0FCY7Q-)EHB7P zK@n)@jAus5G>3_0@3nPG0jGvZg$)exDs_XgxxS2U)tz41BF-%-S(t#CTu}*u3#al5 zDYPH=@>j%|Er#GA-rldyudQxNCxYtW43N!ggn|@HR#7Y$-Kg+rrTdE(>s@yzrx&K( zX&!sq+0VSijWv@$Hv`nc%n~m5F=FeD;-8pU=f1a{%<^&Xp?y?@&Tyjl)zk1SbCX*q zx?}8E!qgST!laB(MqtwGg|%P3eK5r_Cs<)bSnP<+?tkcMSw-C-#0y>%k_O?!@adiv z-j}m^`}QUC->%&X>etoUthS5=r`Dw<7yZF|(GGlK>w?j_($3^{u|J)`qZ~I{uaOTp z#lh&^tk7V(g4WOsa9Y};_J{4cHMfM9uG|J=oE7sG5R`EL{sN?->__*#_U=+TMaxk6 zCIa#V?KTFFIUBfRRGDz7$s2yENhOFR&cLMdU3<4?k3^S5fkjbbOY29(H7NB<$h?Tb zyNKj+-t|Ia1J&mY8&V6CvX~~2h<%LgYn`BIw@UD0riRAHtI!m)h-tA8D7S270e)XW zgsbt*1-u3+mETwfgWqT3Q3Al){Dilh((UQc;s>o*X51Q!rp~RpN*wvmWaHbXam$0sGktL0;Z`F=A2N-AhKF_Pv>2``ii7r{1eF}rK&hXc%!;qCzI-Mwlu2nfe9cHuJE%Z-5<{*7>%Q5|0 zXD*+j;H5ZuyfSUyBe}S3{FKlb4*>Ts@yDV{X|s2lnLNV$Cz%8pLIet8ZST9H*cHN$ zlcy1ZD_SYGss6g+a@8s_Gmhb3fyi7>D1%kPBHpaD%5`Defj_^Jvf3N^BG+Jy&gPb3 zb9a*36WCB;J+|3(&4UkWoSx-8HFINYceB#?yAv}~{t&K3YO~?t zmSF$pvls~7;s_&0jSHJxP~c(4IDyrA9>ZtzJ7rSaar*gKK=ka}G!+FX``GtGVPO$% z`zj!wp=C3TIT)4~mScBC^f{Uw?J(vcmtZhk1bFK{nj9Mb!m=Xxi&(H1z8xIb&AycE zoC@oX&#p~tW#OVq4!X<0Wtnn8F3QzXI}LFj@F$(avC9iB$1qno&{Mzb!!_Rc76xjt zHQFU85ZC{@_~upuHViV$&!c|Z!qkD0{8n>QP@pghPCgKtcrjs85OgnNHz70gJYY4B z&(@^~jrk-UO3&nR6VSxKJKqr_LF}>g@pd$+8|Qvl$))EG!C8UIl%kU0=->#x!fb|7 z>E)%{zV0(7mxb!|&|Y}qBd&P;?zYCTWapH;1Kh{uLmbslWXx4@;K=K`zIJf!n@?VT zkxQ>on5@dGjdDd{H)Q@ziO^H7>pie#4zd zogok~<52hj1_*>VV&d#cy6YqPhTIFz56&brkIC$L8OP@} zv1O35VPDPXRJ!TRc!q6bI{XKeeWi+SUl$hF3SJx1q(BA1C1IdaaNJ{+hkQeI&rBU7stmHR1i3y=7g^DXWP->%~{&5tEeN>OXWK<&w8M6^*$_%JHa8l@ZU$&Ut^C;^~d ziWULv(%W|c6Xa5q0wTfzUws0(2huSpQ0!Rhu&Ers~tSxUZgARK7xF#T` zEvf}j<$^J5U?Pa(b+=Q_EY(;rQ+qGHwqO zBa$4yvgA9_U|krtD*(E&lOeveqnyP~!RPi8C!b-Um~diaQ3Q-|JDJ5m2oJ`VW-GhTp6Vo^oM~u)go6fMnf&;AdNl4X`$GXwIEN*)Sny+sJVGno z`ZpFs{5_yyle`Ztbm-%Gr1(BY*>Lm_S7ty|sADsnJ<2|Uy-uY_{!yjNBzU!zZWl#l zAYS2P9#?rndUWI`q-YqiMp>CMQDyu=!njDSF~2=BRU##=KhG5cJ#4d0aeeC)%az2u z^#1}@f4}DXSl3P`0Wt%^-VlpS#zH|l-t?WUNX&2=)3celm}pf!(D}>#rtZ6t*GdR=c`^Zbc5xBqbVM4mmg*pT(^+Miv3Bj79~2~ z29VTN21>ec!1y~pm(>qV5p8z6?JK(qm#co2Njj3QT(7mZRFCZma^3xx$G;;I+_K)KdS=f2H~C+}W|Vv@ylYi(GkBl1DGtJ25>iw5m;3c! zhu8ufNoZ3kSL*4WL1d-Oc>SnGV+s@zsIsKO1{5jC7h$XSjg`0`ArWgc1b&6LhlG}Pn&|mCQ2Wzh z3AjZY_~k_C6ecx|BO(|}|JM@sx+_uo%XaxT^zv+lmacAUAn}PB1t-%77xf7NOf>mC zE(TB{2lne1>Nsd29SDRtg!tiaZ>mu=MYfacjD~GgGO~Qr?@!@3QkJbh;5J0i0~CT$2a0c_>}T_hVFb z5n)P#6aSqmf03#%p|3qQmL0NTmm=lbwQs?#QQ=Ah{rbuAkgu6n*^yv~ zdYPg?gtJh{hP-4I?;`%zFMWQ204w6XvfCi;{|Q^&=e{l9+bO4j%pQ5>hC{Eb!($R~ zDSgUAu9qj4ndwE>x=7OJ*_1k4N@zi!YhVN!@$i9uprMO6D79fCM0 z{d{Danwqm%bf=qvUkKBuq*79?Krua=ZFEWJ9A(az7dbZVs7zEfOZ96$L zPoj^h(1u|-{hGCSK<>9+i=gY2rfly{P`XZ;c|NR{+G2IT%pq~nAQ=w?kE+4O=Qm_w z(Xyqaq^W9m%AlKP6&MIu`sehNWeM1Sc^d8p*LXq;YzGK-!G_rdZAsal%IZaMX8!nSUI^yNH4{K_ zggxUXJ$8<7rzXiOWEZyxu}c0ijtkKIAm+8G*b!uC`e4xlU~51UaXt5<>?j`1YzgNY z=Op)v>rSXTil7TL1#n~z^95&E`}6gEi-U?~!B0)noLx->p4*@X+!?nZQ{DQSL0DOR z33>UdYK56+CjGkpO8qm)7i5IZPy3XkC#Gzj2TFc}nod{A)VR}9@9mx%B;!&;1xFlO z`{f+(@c9sW(SR$am5EnGlbA1HS)fgog=iBzQ~}TA!js&U$@gU@#ZP&vJpmj&{I!Bg zL?Ct#Lt#q!)+^)73nfMYYk2gnEP=@i#>U>h@~c~*x_g<)vl~4Pa)LiQnYXh2I9^*U zYDt)&mU!*Rec*CQ$>Z5OUm<6!gR$96pO<%%R7Pu>Pz!Llu#-q$q8yhzXwKIYf0RBn z*i&5pXomSkm*&|uNxxQowrq^{A@7=*i_Sf;gXyC*4Xo?zK%+Be4Z@y$3N@hm$cDH= zz1uwu8lD>y=`ZLH>9uaNHZ78segy{jbaZ=!`D~(PmZaN!VM!z<_>4SeLIw8Y$0&p+ z!&*(#XuGE`P@?cIuUk6&B(Q^tu-i5#xsewNrE;&wSCa{;dmJ(-pkcs6XT@B;bY9@7xmKSe&?Q z8;}_BUI{3wC4Z;YUD@~Alz=L2|JS+~9Kvuy5rU$6xTf*XuZ&~ELs94hW*wvG0^#Z$ z_7)3DWEqndZP4xX#>-kN*!zs*ZgHb>V~Awa1MR@pdENNet@lUXEZ@7uu7-NG7Iige zs7izOE3D>t__x#s-bLdeiJjfuGitX@+Cf0{jNgZz`%&0hV3iBv<*GDFo#Z&T|8JVC zZlUjKWK5_UX?SY%EvNH0<7x})>>%xMNK`QtDmHxu7fI<3Uh8SkE|_iRa=-fZ@4@MdV1FE*{NHSTRNpuYg&FDo%B^?RgaQ!^>1oXS*2SB_bRY@YaU>iiNHpLt1n;f?J}^kMWvvRB&9$~_y~E*=J9_(hgOo0GM2RJ{o+nk zF+9$|J!n&adUI7FzRY|OugXjNgq2CiNFk= zNtIe-b2z4X7sER`nJJ>(VMzDfRu{`v$iwVIktY=ep;LbBaTb1J(^I-xoY#oW$fQCo z@@pPV4nO8G1`Yuo5m(&%R0p0WGako*4MuF52XET>U_L5-C_sEOY(i~6dF6y;1HN6S zRu)a*+RoK>&Gw0c{gZvtR>g?gO{ixl5;mj8>JIvG(=7;|L0JXY{^(YS7Nf8C7vJW` zcoaXEfU#BeYpU?NVG#6}9{vNGBzR$9GSv+F06eSP`05)aEuNEF<#SpO4oBI3>1TBZ z8|Y*i@9ooAXD~;F5xgW&q?wNn1_n-fF+#gcvoh1%@>F(+iA~;oMKGVu zY&^D8Pi|&zXxWpBcRXeO5#!-W9lz`miMoDy6#35Me#?!$r`1VEjNbUC!}^Ws)&o1= z+A6a4Z0k)>Q7~mN?bo|!~WgsPN`gl(p4gth+5 zwNxVb=SM7Q+Qhq}9}EUFBnwuk?@!}!l5suBD3fD74qKDZ5-Nrz^~n6JeuW;h%y-GL zm$GxLwGMLfNDJDT{Ph%letm#2Upq$#Rrm=Y5Ui9s6Myu#_u$N9t!Q8l4dO^`(Vrr)w#-p z*ind_b4~f!we8atf(`nEqlbb7XWeA_!dT+M%finV_nMso7Yb{s*!X2vlFfZwR`Re+O5OY#pM@1o3@^!N;6H z7=Jd*Iu|YehE9fWCn)Mzc%1fwm>YukZM^Qr>&AgAHUD<&chu9j-!WC~*M+KRps3s? z5Kn}Ae)M7<+ipqW=b&AE)?A{1NfT?fXU0O)p1#2y^wm|*3_>yUcA|LRwdGU*<2!-U zW10vQbR>acEjrRUk!kJhI)Z7QZx=y2|MRy^mzPOqg-79Ip*0U^s0*8OjYpxaM{`_J z{YD|d5;0+Jl6=Shn^4#IbAnaAZ&f=y4}=+{=ImutArkjy#EZkwP8HN+&SkW%8vH_a z#+yB!@u07@M?^!fR1ZoF@a9V+SsvUBW$~R}Sk>WSDg@^xclLdnK(!7(0-M$P~7Qa)6HqqCzDWf*xj|AUVs|Wm^ zabANCcfHX@`yRu+!7Ey0(phRND)7ht07C16UtYzB`wk}ouiRjh z%h|`%01uxlaurX|?`CEz8g?B6m58xI&rz)PtYagG5`#*;e9=G|_)Gh%_`@fHrwI%r zEMRSD%Cq}{Fr`_em#REXqyU1UC% za@1_D)>u|mw)FweEf;XGe7H5(G?z#?TD%C3u66rTlG)RN-Wh20X1~+HV|&jvzUg!z z=e4k%c_oFEkPd(J9L3E9|G3Q-w&5|{^I1VYHhB$Gz5Up?X_^pU*LRD2Nijw^I><^k zwmXSG8DK&=da#GM%GoH_E`&}kj+b9ax_iWj*CFF&me5!0hOX(Z1^rDAak85&+n=}s zJTaCn&#&eeD8VOEcKqqcu0yy7z4Uk=GgCHGm#%Dk){x9KBJ3H;(V;ACK)=>t_uS)qw&v-e)^|13mjvy z^lvo2FRU$D*pn%rEoX2SBR4~dl-KUgn+gU)RtaKXAtn$&Kjnw&@!J7AZrm5Icrms9 z+fjf;CVLqX3HjvZI3aaGZRtj#`Vyxep7r=6v~eFlz8p3M(uEX=@^JV*S2c3>b2_;GH{bk7g7uY_ z1wYLIf(#Z%% zsXdAI`sIp;M?q~;Xm9s2j!WyJ*2?%qON;1$4?WzJ+XQYnwM21Ez$=8vZese8>H^JK z@+!09EMQ63{E>vibYfUd)!IknX&;%%%Dkyw%1vT>?GQb5ca+$Bl6*`7816cxo^ z0Tt%@if3C{?Is5KcXWN7V@F_UmzXHV1Lqb9mGgnx$czVS$uY!J)vQ1?eA0T|4K9@E zoBT9n?1iNBNWKg|<~v)eCR8JM#(D8#r1CwLJsq6n-mP-SDY_1CT9Ktm-Wcv&WSSO- z^Fxod0ULdea-<)ODm9oYUP=IDdtIGFw>THxH~O7>;pHuTL~8o{FVDQxU>xDVBoPF- z_$NK?`I}q5e(`19F4P5@jIyZNjx5t0e6&;amc8sMvHfZ$2+ImEWj&BG0x}XXU@IlH z{}LgwlgXngm6a=?h8uo!*90^79c*MXcv2bLrx;L*zq=&IUl%yjz!FZrebc?e_MFB-EafN6$WDim_m*s!4pH1dxy~Z1?uLiKEkxmX( zE{1&%ZmDzEN*IF)fYCp0Dv?aeo*#Q^KL2R z%cT05ayx#T8&_Y<3S$kjuz?Z1s-O7e95{#ug3thlzYqu`XOldiW%GOPZRE1^xMEJqX$%CmvG>YVI2@;4x zA^X)$^>W&-d@sMfSr?hc;k)~yG(t1gVR*}*XiP6Ro2fP5!@bUg?&Grncoc43_66L>5EiF zHLS>PhX+x5+=QbzRfYT;+2Mp*s*}5bIBxKaEm?f?)0nRt8{V%qP8K10F8i2C+1ch} zN!2&&URp(B`;=>N zbStLA%$;T}?`B2N^$%w0l+CXF@g=?H{;V|UnvUSKu8%G>UFvv+;SJp7`=czi)aue! zNvmxRWR>|Xun(9*EqEsBWHPHCZM5ziUoz7WQ$N7Y3k;LpAJf3@*4>=L;|$|PbG0

tC@}ewbwX$@p zDzLiZqZ#g>&kGc+GAxpY3FoZ#B_BJUN<;4kvxA z(U4_xcy#-b&-%y^pJRuNhD;~d-n#a#@z`}96WwNwGQZLwd!rF^s3|w!h02T=ElJd-cva!GUpa&Y^r9 znBLRT&M^DrU}Ez$C2XswrJ}iKtM;AcWe0Tu-?UnzI%b{wnD-P3+pj1Ze>E)s#)6cW zcz>zIgnG2O4>R|>m29DcB=tWNDGl9lRR`*krm+P`0pH-ZnQ1EY5$oGDD3`WkN9^7I zEtbe6GY5m=<;bMTT{m_l^=5tFb<_8Ae4*07j&0Mx@b&SmF;_I?No;=B*Aqx@d9YNN z9P=9pQLx3SSD7E($`n7@#CV)BcA7m5;u6<3e|Je zL{$I_-$U{V&Q}^59yhZiV`@ODMtVgaERj?O(El+?(p#z+al2;G@g)p=xUF>O1Kn>6 zI<|Em?=Q8X){}-j34P$xdqPudf3(2sdcz4OoOnSt2pKM6pd?_i+*a@ zwC)Lyk%rU=|Iu^DaRL3gRjMFllFguF+tTu`Hy9_Zh?N5~7YI-z#mGKa1Sx@-zMFL3 z4#w7sr;4`09{;E`x+qp=`F>?MHauKYoaM9799B!D+h^OhZ#FhM>S*x2)%Y53<8^*< z!+!t!Q5+4j`N~m#JD&8C@HoVQAyq6EP^yMk2hfa+(Gw8gWyXl5#3^vVbO1U1k2@^c zhFJ393>XMFr(Lw068V`KZ*|I+Sc36nH~Q{6zahErkG>p zi131GwaSJMKP4Wr{7$`j{Xzn;y1|6w!1~>$E*Cqh28odOX=0qu-h1^Oy33Sl*d*IB(rWaOJt_rc^f*)2M3CMtf9=04-f_c~JvONG2ZzSt%jEfO z*1P^ATL-*Rcw??OOSrQ`-4FC-7?s>G^~hMD%|$r0ZW#*c=n}5C-Q(8g;`MSFD3}BH zxwY^BP4mUB)Era9zDh`@AS;BE>sjJP<3q zRZ)7>Xw)utxmHAzpXm9|<<`x7HZnx~yt1rngcUhwkDb!qgA2O{fs~^Pk%NfCX zoHs14zbBnDFDBMp9tVu%|FHYRzUn(%)ohJH1xVvDX0j~SsaUJFoW(D04Fkp+&S1sX z@eh<35{r@reWu3?)f={aET{LGm!HE#fE`#d&ShZb2t%_m$M4{)=|aeSbhGvYM!#^v z-&*XMZ1Y2Y*^#5A6yu`M^~k77B?*c~7LbtncG=cC?G~U3nY!a8gbuUsC9EdiW|a^> zl{d1xP<|@CEbSDIS=&r&KfSpI?8G0lS=@9V?Mk86bN&H9^eCKGAzdP~BO0G4JB@%RcSc+FzMV&Lb-Z9;1ah&$ztRNeBruTlZ zS*%)rm}8q9U^q{Z{XTD|ULPXLCHC;;*+PQ}l+( z_yrjh?E%kl0OY-dOo$BTH0%axHkR|pG_0-+-aLt*>XV_SUlido{ezk3aGMvf{$ zGsTpKP9j_n4;-$ynQ5X|aOJ{-=#6SOSpH}c4QX|(HR^*}B-A!^piE4uY|dH)T7zrP z8l6?-Wcm~A^QrAGd03Y!&^M{=@nQv7|8{mI1ov=pAz*E3+P))m3tfu2lN~YXK;UA* z6MSJ=&v)JB!G77!dN)MQe%^b2by2t`A{LizW!1du*zmJ^-~@l$cED*-*PDZZF#Y^7 zg@!H=GN?*-QQ?J*%tUWrJ;Btz%^PqadkKyM6nJ)Syr-n3Yy<^WuB-F(;G4W5Oh+xN zL>9jP@3=Mr17*TEIOOf3?XuCTx{ZiQxrL}OXj*2)D0*Nh>}xNmka_jNMCK`cKfaOu zWbh_*4#m`dnaj|HAURfQf2`rpCm%Qe9Zw`}+@tET1{^MhREVa+1J1*DWb4TTn~RM- zrBcJ|6d^lMj9_)!>l|aTuE;{=2hO-~$9%1J6;4hM zpZ9+jLMQTm=j)(BxR>$ktE+?9`d%Z89WH$ZQ*-79^)WR8DnGVbg{tOH<1-oEoQ=~KP0+wf+SG`=-1%sO4-xusO; z&Fis3kaJ&V(WEBksvRHh9fyxmn}Iv*a5KT6$wyBIuBl9hg`N{jsp_bagZM(HF9f-} zH4<$VQ9RE>+X--#D?vi8%=Gba>egK9yU`1*&Q~n8Z>f=SPi2L7BU&hA*~es2wKbt( zg03)Gjp4K%rvY4Tm$DZvb*;U-rAS$w{I)Zt)NDrD+=+_Gv}Q@DNH%+$e|%|vghxkR zE)0LgP$7C{zj*Jt`Ay`5Ae7hD5-KICQ+sDp!6o%rcg6efXqSWF7VbNU1rTzGbjRIJ zA&;jBY(C_@+b-1MuG#bTjLIdK1gZyk^K}9i#nPi(--IiU0b3sid=bQNSD_ZvtE*?r zPH$|~b}7YHO;zfR`D^ONPCdWmRvFNHWbq7d!q62ct!~bAHrxAb&!_vcd=9h#Pd(5A zmt375#rH=)kG-W}dk5T(*QTil-U@tx1|HZlT)?QS{S3Ec7s<3)EiiFnj@0Tsb@^|| zkhR7X>+v~l?b&$9f9bNccQds)6X1PJbjsMo5~Usm@vazbc+JxPalu%VjU&{+`Q>E4 z(I{v23Iw(7`|vKNE3ufKP)^wvzRFV@B*>fY-CXaUw;X2Gz6F4}C6D^IEu?V-cQV6N zqMv1GE`%VSzFHGNf2tI%oe?g8BMSyK5Vz?%7z$I#Y!MJNHXDM}HFYqNerR)>GM(=gP>OJf^MeI{Azp4t z&s%WK4{*3=EZQq_Cm||Ox1$*3RvlUr1&)H0a*Aqh%>1kVU@e=@GJScI=I^rr{{wHZ z;(BBl=m|NN*U4ce-L4+Uko6iBMw80wz{&IaAI$mF%x;n0a8I|62GO=D9VcO7RH0kH z?x|$=*fzXY(+V1DK6t#nima!Hi2L^RkqUGDffX~#4&WkOUo6#}nhhMdR(tb|k#i;4 zUVz@!yL%Hc7nAJ^Z_DJ>FNGws_RCu$&`46p7HkPn211V%R;w8KYZlLJL6-N+0g*>Y zmfP8~FP3wX2DjI+y#2RMuO^>yjg7EX25Ux5B)-uS!`hu*l$`WyyGkvsBww%>L+tx1 zgx;3I;E#x`s=-XJ^snEANrzD9*lx=Ztp z7eRm4Es!=3N6r>`@GaMw%E+48TuT)#Sh(RXB!4!4`!2VQ2iVYSkMLe6T%`E2$MlLf zy6X}Zbw66ZmSB&|Nou`)(WWdY*mcM#_16WXNcf$fjo6B5LjNob0Nqikqbc#i=A7)a z4S5^#XIoUC`rh+&u8aQD-0YBmFz}}TqhRNe|$JD zte*2sv>yE;V@z*gmlrBTV=d4HOO>Sg6AC+h);n@hnd#4)Zv8||2>b{xSuz~lVwU|j zIT!198vTt(ThO*CVI@|JFvQX$yodF+&VAE%(@_k34l@F*a<2M&&8H;{ICT{UJaZy# zP8`}(kngT1OQ2Sw4=s)fF}yXhdp(X{E(A`0W-Kkgw4$%YkHW0?{6(9^avISL@g>H< z16}He3gpWd81-stB#5VxY}hL|Hev~EhVV@UBqoVciBKI}eEgxoe}pt9oHJK0pHYt* z%Gu7>K~e>;%_sRzXoE z1we^EO)9)VKGS-ax?NlHoTcF;^Kw{HiqLwHy7{nZV08gK{UTS~>v2m`_FI!Cp*1)H z8Hbl*sXjYL$ogZx{MlZJ(6_o#@|4B?5}9R}k1&h1)~cE@^Z10Qn5b@~q8>O6Cl|hN zRj=f}pU|HnG@6+LZ{(PiR7qR~C0Uv@UD@{)r_uP}s_IKVB|LqKP z!?sD+pYg)Z%Qj)Ii8h?8!&A1uKrs>MpzHk>OI0SRecbtL84C#-+uCE!IjK{ZVw})u zm6DofEv!o4wvxw;H^j6OD*p}N;&{QV-AcVOqQD$)tiJvl#pgJaSd;FK@nchOcaY@& zW*V3o?8bh?5>x$9qIoSjZZO@QISQ>3W-TH!e$8j{gO>q~8sTZ0G`H>MhJ~0BUj`*K z%hrf-`(Oa6oD?f^$VGh{hmlBjd*cICNK=81PjDRziRhj1CQC7?!0JxvFVJd|Wl$Hnfl^6&N`m^6jtfHw>+sC-w_PpOQo2LQx-0h8 zd%M$zduaUzh53Ick3AC#%zF=Oq&sCIlKOQ7({p`A#OiANk1_hz*cc#&{nym-fEAbe zWwZqNWVz)j&%s3!q|O<6XMLxX`460!j@n+Hjzo-XApZti)cpU%n8ipiAuy6?F;yv5 z{^JZWk|aNjI86p#ntwuR;M4ru;^4rPzbXr>eff`ze1$NEl>`kR$P4_h!~XmDQXpRiT|&oh2H^%6$Av#O-f8y_5Vl1G9w5~dFV`X7;qMA|Medz{{QI2 zi63ARE)buP>$wrCT@kn3h*B=3B}l=+zG{avnE%%#JcI!~ft$mP(P5&ia8HDT!A$K7 zMOxktgg(Fe**7Ihqbc>DQ~xVOjw28}UeB*OMUb!|O*AeQ0rjtGy+7;z@OItm{*Z<| zF8FJF{-<^G$bv*jcDx{hJ%>Q{A`(F_9WC)j1 z(le(2*Bg^5LIO9&)Ier_{Pmgt(}3|1(5g+w7E=5hDgN&_P!$Gla$)*8YyW3YY~7My ZeIo1Y@24C(ynp~dQsVMrRU(D~{|ldBnOgt= literal 0 HcmV?d00001 diff --git a/docs/zh-CN/DEVELOPER_GUIDE.md b/docs/zh-CN/DEVELOPER_GUIDE.md index c118e03a2f..752779a763 100644 --- a/docs/zh-CN/DEVELOPER_GUIDE.md +++ b/docs/zh-CN/DEVELOPER_GUIDE.md @@ -377,6 +377,11 @@ OdcServer 启动设置示意如下 ![image.png](../en-US/images/idea-run-configuration-start-odc-server-2.png) +环境变量（environment variables）配置示意如下 + +![image.png](../en-US/images/idea-run-configuration-start-odc-server-3.png) + + # 4. 前后端集成和联调 ## 4.1 基于静态资源服务器的前后端联调 From d02afe21f28292358da1a26924edc31970970ce7 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Mon, 25 Nov 2024 14:44:05 +0800 Subject: [PATCH 039/118] feat(session): support Oracle kill session (#3898) * support for oracle kill session * delete unused code * fix code format * delete unused code * fix ut * response to comments * fix ut * add @NotEmpty --- .../session/ConnectConsoleServiceTest.java | 9 - .../core/sql/execute/SessionOperations.java | 30 ++++ .../sql/execute/TestSessionOperations.java | 9 + ..._2_0_0__initialize_version_diff_config.sql | 4 +- .../v2/ConnectSessionController.java | 4 +- .../service/db/session/DBSessionService.java | 40 ----- .../db/session/DefaultDBSessionManage.java | 170 ++++++++---------- ...KillSessionResult.java => KillResult.java} | 4 +- .../session/ConnectConsoleService.java | 37 +--- .../session/DBSessionManageFacade.java | 4 +- .../obmysql/OBMySQLSessionExtension.java | 13 +- .../oboracle/OBOracleSessionExtension.java | 13 +- .../oracle/OracleSessionExtension.java | 14 +- 13 files changed, 161 insertions(+), 190 deletions(-) rename server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/{KillSessionResult.java => KillResult.java} (92%) diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java index d0ec0ce5db..024eb52437 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java @@ -97,15 +97,6 @@ public void getAsyncResult_killSessionSql_successResult() throws Exception { Assert.assertFalse(resultList.isEmpty()); } - @Test - public void killSession_directLink() { - String sql = "kill session 12345"; - injectAsyncJdbcExecutor(JdbcGeneralResult.successResult(SqlTuple.newTuple(sql))); - List jdbcGeneralResults = defaultConnectSessionManage.executeKillSession( - sessionService.nullSafeGet(sessionid), Collections.singletonList(SqlTuple.newTuple(sql)), sql); - Assert.assertFalse(jdbcGeneralResults.isEmpty()); - } - @Test public void getAsyncResult_wrongKillSessionSql_failedResult() throws Exception { String sql = "kill session /*"; diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/execute/SessionOperations.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/execute/SessionOperations.java index 65fc5c0a7f..b3a54fd175 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/execute/SessionOperations.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/execute/SessionOperations.java @@ -16,6 +16,11 @@ package com.oceanbase.odc.core.sql.execute; import java.sql.Connection; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.validation.constraints.NotEmpty; import lombok.NonNull; @@ -30,4 +35,29 @@ public interface SessionOperations { void killQuery(@NonNull Connection connection, @NonNull String connectionId); + String getKillQuerySql(@NonNull String connectionId); + + String getKillSessionSql(@NonNull String connectionId); + + /** + * Get kill query SQL by connectionId + * + * @param connectionIds + * @return the map of connectionId to kill query SQL + */ + default Map getKillQuerySqls(@NotEmpty Set connectionIds) { + return connectionIds.stream().collect(Collectors.toMap(id -> id, this::getKillQuerySql)); + } + + + /** + * Get kill session SQL by connectionId + * + * @param connectionIds + * @return the map of connectionId to kill session SQL + */ + default Map getKillSessionSqls(@NotEmpty Set connectionIds) { + return connectionIds.stream().collect(Collectors.toMap(id -> id, this::getKillSessionSql)); + } + } diff --git a/server/odc-core/src/test/java/com/oceanbase/odc/core/sql/execute/TestSessionOperations.java b/server/odc-core/src/test/java/com/oceanbase/odc/core/sql/execute/TestSessionOperations.java index 023a4ee67f..68c73a3721 100644 --- a/server/odc-core/src/test/java/com/oceanbase/odc/core/sql/execute/TestSessionOperations.java +++ b/server/odc-core/src/test/java/com/oceanbase/odc/core/sql/execute/TestSessionOperations.java @@ -50,4 +50,13 @@ public void killQuery(@NonNull Connection connection, @NonNull String connection } } + @Override + public String getKillQuerySql(@NonNull String connectionId) { + return "KILL QUERY " + connectionId; + } + + @Override + public String getKillSessionSql(@NonNull String connectionId) { + return "KILL " + connectionId; + } } diff --git a/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql b/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql index 80d5e661ce..ce35102a64 100644 --- a/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql +++ b/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql @@ -262,4 +262,6 @@ insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min -- supports ob external table insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_external_table', 'OB_MYSQL', 'true', '4.3.2', CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_key`=`config_key`; -insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_external_table', 'OB_ORACLE', 'true', '4.3.2', CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_key`=`config_key`; \ No newline at end of file +insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_external_table', 'OB_ORACLE', 'true', '4.3.2', CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_key`=`config_key`; + +insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_kill_session','ORACLE','true','0',CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_value`=VALUES(`config_value`); diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java index daac207cf9..2ceb511946 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java @@ -47,8 +47,8 @@ import com.oceanbase.odc.service.connection.model.DBSessionResp; import com.oceanbase.odc.service.connection.model.MultiSessionsReq; import com.oceanbase.odc.service.db.session.DBSessionService; +import com.oceanbase.odc.service.db.session.KillResult; import com.oceanbase.odc.service.db.session.KillSessionOrQueryReq; -import com.oceanbase.odc.service.db.session.KillSessionResult; import com.oceanbase.odc.service.dml.ValueEncodeType; import com.oceanbase.odc.service.partitionplan.PartitionPlanService; import com.oceanbase.odc.service.partitionplan.model.PartitionPlanPreViewResp; @@ -213,7 +213,7 @@ public SuccessResponse killQuery(@PathVariable String sessionId) { */ @ApiOperation(value = "kill session", notes = "终止会话接口") @RequestMapping(value = "/sessions/killSession", method = RequestMethod.POST) - public SuccessResponse> killSession(@RequestBody KillSessionOrQueryReq req) { + public SuccessResponse> killSession(@RequestBody KillSessionOrQueryReq req) { return Responses.success(consoleService.killSessionOrQuery(req)); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DBSessionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DBSessionService.java index 8cb42bc495..9b0e4b25d2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DBSessionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DBSessionService.java @@ -15,10 +15,7 @@ */ package com.oceanbase.odc.service.db.session; -import static com.oceanbase.odc.service.db.session.KillSessionOrQueryReq.KILL_QUERY_TYPE; - import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -26,22 +23,17 @@ import org.springframework.jdbc.core.JdbcOperations; import org.springframework.stereotype.Service; -import com.google.common.base.MoreObjects; import com.oceanbase.odc.common.util.ExceptionUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionConstants; import com.oceanbase.odc.core.session.ConnectionSessionUtil; -import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.shared.model.OdcDBSession; import com.oceanbase.odc.core.sql.util.OdcDBSessionRowMapper; import com.oceanbase.odc.service.db.browser.DBStatsAccessors; import com.oceanbase.tools.dbbrowser.model.DBSession; import com.oceanbase.tools.dbbrowser.stats.DBStatsAccessor; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -73,36 +65,4 @@ public List list(@NonNull ConnectionSession session) { JdbcOperations jdbcOperations = session.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY); return jdbcOperations.query("SHOW FULL PROCESSLIST", new OdcDBSessionRowMapper()); } - - public List getKillSql(@NonNull ConnectionSession session, @NonNull List sessionIds, - String closeType) { - List allSession = list(session); - Map sessionId2SvrpIp = - allSession.stream().collect( - Collectors.toMap(OdcDBSession::getSessionId, - s -> MoreObjects.firstNonNull(s.getSvrIp(), ""))); - return sessionIds.stream().map(sid -> { - PreConditions.notNegative(Long.parseLong(sid), "sessionId"); - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append("kill "); - if (KILL_QUERY_TYPE.equalsIgnoreCase(closeType)) { - sqlBuilder.append("query "); - } - sqlBuilder.append(sid); - if (sessionId2SvrpIp.get(sid) != null) { - sqlBuilder.append(" /*").append(sessionId2SvrpIp.get(sid)).append("*/"); - } - return new SessionIdKillSql(sid, sqlBuilder.append(";").toString()); - }).collect(Collectors.toList()); - } - - - @AllArgsConstructor - @NoArgsConstructor - @Data - public static class SessionIdKillSql { - private String sessionId; - private String killSql; - } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index 7b9a890350..cfebed9a27 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -20,6 +20,7 @@ import java.sql.Connection; import java.sql.Statement; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -38,9 +39,11 @@ import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.SetUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.VersionUtils; @@ -55,22 +58,20 @@ import com.oceanbase.odc.core.shared.model.OdcDBSession; import com.oceanbase.odc.core.sql.execute.model.JdbcGeneralResult; import com.oceanbase.odc.core.sql.execute.model.SqlTuple; +import com.oceanbase.odc.plugin.connect.api.SessionExtensionPoint; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.model.ConnectionConfig; -import com.oceanbase.odc.service.connection.util.ConnectionInfoUtil; import com.oceanbase.odc.service.connection.util.ConnectionMapper; import com.oceanbase.odc.service.db.browser.DBStatsAccessors; -import com.oceanbase.odc.service.db.session.DBSessionService.SessionIdKillSql; +import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.session.DBSessionManageFacade; import com.oceanbase.odc.service.session.OdcStatementCallBack; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; -import com.oceanbase.odc.service.session.factory.DruidDataSourceFactory; import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import lombok.NonNull; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -97,7 +98,7 @@ public class DefaultDBSessionManage implements DBSessionManageFacade { @Override @SkipAuthorize - public List killSessionOrQuery(KillSessionOrQueryReq request) { + public List killSessionOrQuery(KillSessionOrQueryReq request) { Verify.notNull(request.getSessionIds(), "session can not be null"); Verify.notNull(request.getDatasourceId(), "connection session id can not be null"); ConnectionConfig connectionConfig = connectionService.getForConnect(Long.valueOf(request.getDatasourceId())); @@ -105,9 +106,15 @@ public List killSessionOrQuery(KillSessionOrQueryReq request) ConnectionSession connectionSession = null; try { connectionSession = factory.generateSession(); - List session = - dbSessionService.getKillSql(connectionSession, request.getSessionIds(), request.getKillType()); - return doKillSessionOrQuery(connectionSession, session); + Map connectionId2KillSql; + SessionExtensionPoint sessionExtension = + ConnectionPluginUtil.getSessionExtension(connectionConfig.getDialectType()); + if (KillSessionOrQueryReq.KILL_QUERY_TYPE.equals(request.getKillType())) { + connectionId2KillSql = sessionExtension.getKillQuerySqls(new HashSet<>(request.getSessionIds())); + } else { + connectionId2KillSql = sessionExtension.getKillSessionSqls(new HashSet<>(request.getSessionIds())); + } + return doKill(connectionSession, connectionId2KillSql); } catch (Exception e) { log.info("kill session failed,datasourceId#{}", request.getDatasourceId(), e); throw e; @@ -134,19 +141,12 @@ public boolean killConsoleQuery(ConnectionSession session) { Verify.notNull(connectionId, "ConnectionId"); ConnectionConfig conn = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(session); Verify.notNull(conn, "ConnectionConfig"); - DruidDataSourceFactory factory = new DruidDataSourceFactory(conn); - try { - ConnectionInfoUtil.killQuery(connectionId, factory, session.getDialectType()); - } catch (Exception e) { - if (session.getDialectType().isOceanbase()) { - ConnectionSessionUtil.killQueryByDirectConnect(connectionId, factory); - log.info("Kill query by direct connect succeed, connectionId={}", connectionId); - } else { - log.warn("Kill query occur error, connectionId={}", connectionId, e); - return false; - } - } - return true; + SessionExtensionPoint sessionExtension = + ConnectionPluginUtil.getSessionExtension(conn.getDialectType()); + Map connectionId2KillSql = sessionExtension.getKillQuerySqls(SetUtils.hashSet(connectionId)); + List results = doKill(session, connectionId2KillSql); + Verify.singleton(results, "killResults"); + return results.get(0).isKilled(); } @Override @@ -164,12 +164,7 @@ public void killAllSessions(ConnectionSession connectionSession, Predicate doKillAllSessions( - list, - connectionSession, - executor), - lockTableTimeOutSeconds); + waitingForResult(() -> doKillAllSessions(list, connectionSession, executor), lockTableTimeOutSeconds); } finally { try { executor.shutdownNow(); @@ -184,50 +179,42 @@ public void killAllSessions(ConnectionSession connectionSession, Predicate getSessionList(ConnectionSession connectionSession, Predicate filter) { - return DBStatsAccessors.create(connectionSession) - .listAllSessions() - .stream() - .map(OdcDBSession::from) - .filter(filter == null ? a -> true : filter) - .collect(Collectors.toList()); - } + private List doKill(ConnectionSession session, Map connectionId2KillSqls) { + List sqlTupleSessionIds = connectionId2KillSqls.entrySet().stream().map(entry -> { + String connectionId = entry.getKey(); + String killSql = entry.getValue(); + return new SqlTupleSessionId(SqlTuple.newTuple(killSql), connectionId); + }).collect(Collectors.toList()); - private List doKillSessionOrQuery( - ConnectionSession connectionSession, List killSessionSqls) { - List sqlTupleSessionIds = killSessionSqls.stream().map( - s -> new SqlTupleSessionId(SqlTuple.newTuple(s.getKillSql()), s.getSessionId())) - .collect(Collectors.toList()); Map sqlId2SessionId = sqlTupleSessionIds.stream().collect( Collectors.toMap(s -> s.getSqlTuple().getSqlId(), SqlTupleSessionId::getSessionId)); - List sqlTuples = - sqlTupleSessionIds.stream().map(SqlTupleSessionId::getSqlTuple).collect(Collectors.toList()); - List jdbcGeneralResults = - executeKillSession(connectionSession, sqlTuples, sqlTuples.toString()); + List jdbcGeneralResults = executeSqls(session, sqlTupleSessionIds.stream() + .map(SqlTupleSessionId::getSqlTuple) + .collect(Collectors.toList())); + if (session.getDialectType().isOceanbase()) { + jdbcGeneralResults = additionalKillIfNecessary(session, jdbcGeneralResults, sqlTupleSessionIds); + } return jdbcGeneralResults.stream() - .map(res -> new KillSessionResult(res, sqlId2SessionId.get(res.getSqlTuple().getSqlId()))) + .map(res -> new KillResult(res, sqlId2SessionId.get(res.getSqlTuple().getSqlId()))) .collect(Collectors.toList()); } - @SkipAuthorize("odc internal usage") - public List executeKillSession(ConnectionSession connectionSession, List sqlTuples, - String sqlScript) { - List results = executeKillCommands(connectionSession, sqlTuples, sqlScript); - if (connectionSession.getDialectType() == DialectType.OB_MYSQL - || connectionSession.getDialectType() == DialectType.OB_ORACLE) { - return processResults(connectionSession, results); - } - return results; + private List getSessionList(ConnectionSession connectionSession, Predicate filter) { + return DBStatsAccessors.create(connectionSession) + .listAllSessions() + .stream() + .map(OdcDBSession::from) + .filter(filter == null ? a -> true : filter) + .collect(Collectors.toList()); } - private List executeKillCommands(ConnectionSession connectionSession, List sqlTuples, - String sqlScript) { + private List executeSqls(ConnectionSession connectionSession, List sqlTuples) { List results = connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) .execute(new OdcStatementCallBack(sqlTuples, connectionSession, true, null, false)); if (results == null) { - log.warn("Execution of the kill session command failed with unknown error, sql={}", sqlScript); + log.warn("Execution of the kill session command failed with unknown error, sql={}", sqlTuples); throw new InternalServerError("Unknown error"); } return results; @@ -241,8 +228,15 @@ private List executeKillCommands(ConnectionSession connection * @param results * @return */ - private List processResults(ConnectionSession connectionSession, - List results) { + private List additionalKillIfNecessary(ConnectionSession connectionSession, + List results, List sqlTupleSessionIds) { + Map sessionId2SvrAddr = + getSessionList(connectionSession, null).stream().collect( + Collectors.toMap(OdcDBSession::getSessionId, + s -> extractServerAddress(MoreObjects.firstNonNull(s.getSvrIp(), "")))); + Map sqlId2SessionId = sqlTupleSessionIds.stream().collect( + Collectors.toMap(s -> s.getSqlTuple().getSqlId(), SqlTupleSessionId::getSessionId)); + Boolean isDirectedOBServer = isObServerDirected(connectionSession); String obProxyVersion = getObProxyVersion(connectionSession, isDirectedOBServer); String obVersion = ConnectionSessionUtil.getVersion(connectionSession); @@ -257,7 +251,9 @@ private List processResults(ConnectionSession connectionSessi if (isUnknownThreadIdError(e)) { jdbcGeneralResult = handleUnknownThreadIdError(connectionSession, jdbcGeneralResult, isDirectedOBServer, - isEnabledGlobalClientSession, isSupportedOracleModeKillSession); + isEnabledGlobalClientSession, isSupportedOracleModeKillSession, + sessionId2SvrAddr.getOrDefault( + sqlId2SessionId.get(jdbcGeneralResult.getSqlTuple().getSqlId()), null)); } else { log.warn("Failed to execute sql in kill session scenario, sqlTuple={}", jdbcGeneralResult.getSqlTuple(), e); @@ -362,7 +358,8 @@ private boolean isUnknownThreadIdError(Exception e) { private JdbcGeneralResult handleUnknownThreadIdError(ConnectionSession connectionSession, JdbcGeneralResult jdbcGeneralResult, Boolean isDirectedOBServer, - boolean isEnabledGlobalClientSession, boolean isSupportedOracleModeKillSession) { + boolean isEnabledGlobalClientSession, boolean isSupportedOracleModeKillSession, + ServerAddress directServerAddress) { if (Boolean.TRUE.equals(isDirectedOBServer)) { log.info("The current connection mode is directing observer, return error result directly"); return jdbcGeneralResult; @@ -376,7 +373,7 @@ private JdbcGeneralResult handleUnknownThreadIdError(ConnectionSession connectio jdbcGeneralResult.getSqlTuple()); } return tryKillSessionViaDirectConnectObServer(connectionSession, jdbcGeneralResult, - jdbcGeneralResult.getSqlTuple()); + jdbcGeneralResult.getSqlTuple(), directServerAddress); } private boolean isOracleModeKillSessionSupported(String obVersion, ConnectionSession connectionSession) { @@ -428,10 +425,10 @@ private JdbcGeneralResult tryKillSessionByAnonymousBlock(ConnectionSession conne * @return */ private JdbcGeneralResult tryKillSessionViaDirectConnectObServer(ConnectionSession connectionSession, - JdbcGeneralResult jdbcGeneralResult, SqlTuple sqlTuple) { + JdbcGeneralResult jdbcGeneralResult, SqlTuple sqlTuple, ServerAddress serverAddress) { try { log.info("Kill query/session Unknown thread id error, try direct connect observer"); - directLinkServerAndExecute(sqlTuple.getExecutedSql(), connectionSession); + directLinkServerAndExecute(sqlTuple.getExecutedSql(), connectionSession, serverAddress); return JdbcGeneralResult.successResult(sqlTuple); } catch (Exception e) { log.warn("Failed to direct connect observer {}", e.getMessage()); @@ -439,9 +436,8 @@ private JdbcGeneralResult tryKillSessionViaDirectConnectObServer(ConnectionSessi } } - private void directLinkServerAndExecute(String sql, ConnectionSession session) + private void directLinkServerAndExecute(String sql, ConnectionSession session, ServerAddress serverAddress) throws Exception { - ServerAddress serverAddress = extractServerAddress(sql); if (Objects.isNull(serverAddress)) { throw new Exception("ServerAddress not found"); } @@ -461,27 +457,22 @@ private void directLinkServerAndExecute(String sql, ConnectionSession session) } } - ServerAddress extractServerAddress(String sql) { - String removeBlank = org.apache.commons.lang3.StringUtils.replace(sql, "\\s+", ""); - if (org.apache.commons.lang3.StringUtils.isBlank(removeBlank)) { - log.debug("unable to extract server address, sql was empty"); - return null; - } - int startPos = org.apache.commons.lang3.StringUtils.indexOf(removeBlank, "/*"); - if (-1 == startPos) { - log.debug("unable to extract server address, no comment found"); + // extract text(query from the dictionary) to server address(ip, port) + // the text is expected be like 0.0.0.0:8888 + private ServerAddress extractServerAddress(String text) { + String trimmed = StringUtils.trim(text); + if (StringUtils.isBlank(trimmed)) { + log.info("unable to extract server address, text is empty"); return null; } - String subStr = org.apache.commons.lang3.StringUtils.substring(removeBlank, startPos); - Matcher matcher = SERVER_PATTERN.matcher(subStr); + Matcher matcher = SERVER_PATTERN.matcher(trimmed); if (!matcher.matches()) { log.info("unable to extract server address, does not match pattern"); return null; } String ipAddress = matcher.group("ip"); String port = matcher.group("port"); - if (org.apache.commons.lang3.StringUtils.isEmpty(ipAddress) - || org.apache.commons.lang3.StringUtils.isEmpty(port)) { + if (StringUtils.isEmpty(ipAddress) || StringUtils.isEmpty(port)) { log.info("unable to extract server address, ipAddress={}, port={}", ipAddress, port); return null; } @@ -490,17 +481,19 @@ ServerAddress extractServerAddress(String sql) { private CompletableFuture doKillAllSessions(List list, ConnectionSession connectionSession, Executor executor) { - return CompletableFuture.supplyAsync((Supplier) () -> { Lists.partition(list, 1024) .forEach(sessionList -> { - doKillSessionOrQuery(connectionSession, getKillSql(sessionList)); + SessionExtensionPoint sessionExtension = + ConnectionPluginUtil.getSessionExtension(connectionSession.getDialectType()); + Map connectionId2KillSql = sessionExtension.getKillQuerySqls( + sessionList.stream().map(OdcDBSession::getSessionId).collect(Collectors.toSet())); + doKill(connectionSession, connectionId2KillSql); }); return null; }, executor).exceptionally(ex -> { throw new CompletionException(ex); }); - } private void waitingForResult(Supplier> completableFutureSupplier, @@ -516,19 +509,6 @@ private void waitingForResult(Supplier> completableFutu } } - private List getKillSql(@NonNull List allSession) { - return allSession.stream() - .map(dbSession -> { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append("kill "); - sqlBuilder.append(dbSession.getSessionId()); - if (dbSession.getSvrIp() != null) { - sqlBuilder.append(" /*").append(dbSession.getSvrIp()).append("*/"); - } - return new SessionIdKillSql(dbSession.getSessionId(), sqlBuilder.append(";").toString()); - }).collect(Collectors.toList()); - } - @Data static class ServerAddress { String ipAddress; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillSessionResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillResult.java similarity index 92% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillSessionResult.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillResult.java index 8c38cd9ec2..26bb2c966c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillSessionResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillResult.java @@ -24,12 +24,12 @@ @Data @AllArgsConstructor -public class KillSessionResult { +public class KillResult { private String sessionId; private boolean killed; private String errorMessage; - public KillSessionResult(JdbcGeneralResult jdbcGeneralResult, String sessionId) { + public KillResult(JdbcGeneralResult jdbcGeneralResult, String sessionId) { this.sessionId = sessionId; this.killed = jdbcGeneralResult.getStatus().equals(SqlExecuteStatus.SUCCESS); if (!killed) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java index 8a19acdd75..f0b0526e6b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java @@ -83,8 +83,8 @@ import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.db.browser.DBSchemaAccessors; import com.oceanbase.odc.service.db.session.DefaultDBSessionManage; +import com.oceanbase.odc.service.db.session.KillResult; import com.oceanbase.odc.service.db.session.KillSessionOrQueryReq; -import com.oceanbase.odc.service.db.session.KillSessionResult; import com.oceanbase.odc.service.dml.ValueEncodeType; import com.oceanbase.odc.service.feature.AllFeatures; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; @@ -246,10 +246,7 @@ public SqlAsyncExecuteResp streamExecute(@NotNull String sessionId, PreConditions.lessThanOrEqualTo("sqlLength", LimitMetric.SQL_LENGTH, StringUtils.length(request.getSql()), maxSqlLength); } - SqlAsyncExecuteResp result = filterKillSession(connectionSession, request); - if (result != null) { - return result; - } + List sqls = request.ifSplitSqls() ? SqlUtils.splitWithOffset(connectionSession, request.getSql(), sessionProperties.isOracleRemoveCommentPrefix()) @@ -505,7 +502,7 @@ public boolean killCurrentQuery(@NotNull String sessionId) { } @SkipAuthorize - public List killSessionOrQuery(KillSessionOrQueryReq request) { + public List killSessionOrQuery(KillSessionOrQueryReq request) { if (!connectionService.checkPermission( Long.valueOf(request.getDatasourceId()), Collections.singletonList("update"))) { throw new AccessDeniedException(); @@ -544,34 +541,6 @@ private boolean validateSqlSemantics(String sql, ConnectionSession session) { return false; } - /** - * for some special sql execution(eg. kill session). This will be required to connect to specific - * observer - * - * @param connectionSession connection engine - * @param request odc sql object - * @return result of sql execution - */ - private SqlAsyncExecuteResp filterKillSession(ConnectionSession connectionSession, SqlAsyncExecuteReq request) - throws Exception { - String sqlScript = request.getSql().trim().toLowerCase(); - if (!sqlScript.startsWith("kill ") || !sqlScript.contains("/*")) { - return null; - } - List sqlTuples = SqlTuple.newTuples( - Arrays.stream(sqlScript.split(";")).filter(StringUtils::isNotBlank).collect(Collectors.toList())); - List results = - defaultDbSessionManage.executeKillSession(connectionSession, sqlTuples, sqlScript); - - AsyncExecuteContext executeContext = - new AsyncExecuteContext(sqlTuples, new HashMap<>()); - Future> successFuture = FutureResult.successResult(results); - executeContext.setFuture(successFuture); - executeContext.addSqlExecutionResults(successFuture.get()); - String id = ConnectionSessionUtil.setExecuteContext(connectionSession, executeContext); - return SqlAsyncExecuteResp.newSqlAsyncExecuteResp(id, sqlTuples); - } - private SqlExecuteResult generateResult(@NonNull ConnectionSession connectionSession, @NonNull JdbcGeneralResult generalResult, @NonNull Map cxt) { SqlExecuteResult result = new SqlExecuteResult(generalResult); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/DBSessionManageFacade.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/DBSessionManageFacade.java index 8720cc17de..adbd554d8f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/DBSessionManageFacade.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/DBSessionManageFacade.java @@ -20,12 +20,12 @@ import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.shared.model.OdcDBSession; +import com.oceanbase.odc.service.db.session.KillResult; import com.oceanbase.odc.service.db.session.KillSessionOrQueryReq; -import com.oceanbase.odc.service.db.session.KillSessionResult; public interface DBSessionManageFacade { - List killSessionOrQuery(KillSessionOrQueryReq request); + List killSessionOrQuery(KillSessionOrQueryReq request); boolean supportKillConsoleQuery(ConnectionSession session); diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java index fb4b428ced..2c0002ec08 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java @@ -31,6 +31,7 @@ import com.oceanbase.odc.plugin.connect.model.DBClientInfo; import com.oceanbase.tools.dbbrowser.util.VersionUtils; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; /** @@ -46,7 +47,17 @@ public class OBMySQLSessionExtension implements SessionExtensionPoint { @Override public void killQuery(Connection connection, String connectionId) { - JdbcOperationsUtil.getJdbcOperations(connection).execute("KILL QUERY " + connectionId); + JdbcOperationsUtil.getJdbcOperations(connection).execute(getKillQuerySql(connectionId)); + } + + @Override + public String getKillQuerySql(@NonNull String connectionId) { + return "KILL QUERY " + connectionId; + } + + @Override + public String getKillSessionSql(@NonNull String connectionId) { + return "KILL " + connectionId; } @Override diff --git a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java index 2696b256a7..e20da0282c 100644 --- a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java +++ b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java @@ -32,6 +32,7 @@ import com.oceanbase.odc.plugin.connect.model.DBClientInfo; import com.oceanbase.tools.dbbrowser.util.VersionUtils; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; /** @@ -47,7 +48,17 @@ public class OBOracleSessionExtension implements SessionExtensionPoint { @Override public void killQuery(Connection connection, String connectionId) { - JdbcOperationsUtil.getJdbcOperations(connection).execute("KILL QUERY " + connectionId); + JdbcOperationsUtil.getJdbcOperations(connection).execute(getKillQuerySql(connectionId)); + } + + @Override + public String getKillQuerySql(@NonNull String connectionId) { + return "KILL QUERY " + connectionId; + } + + @Override + public String getKillSessionSql(@NonNull String connectionId) { + return "KILL " + connectionId; } @Override diff --git a/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java b/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java index a702def74c..0d0d659a4a 100644 --- a/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java +++ b/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java @@ -42,7 +42,17 @@ public class OracleSessionExtension extends OBOracleSessionExtension { @Override public void killQuery(Connection connection, String connectionId) { - JdbcOperationsUtil.getJdbcOperations(connection).execute("ALTER SYSTEM KILL SESSION '" + connectionId + "'"); + JdbcOperationsUtil.getJdbcOperations(connection).execute(getKillQuerySql(connectionId)); + } + + @Override + public String getKillQuerySql(@NonNull String connectionId) { + return this.getKillSessionSql(connectionId); + } + + @Override + public String getKillSessionSql(@NonNull String connectionId) { + return "ALTER SYSTEM KILL SESSION '" + connectionId + "'"; } @Override @@ -113,6 +123,4 @@ public boolean setClientInfo(Connection connection, DBClientInfo clientInfo) { return false; } } - - } From 2178b303cd67eafec6f00817e53ea29a87df46b6 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Mon, 25 Nov 2024 15:30:48 +0800 Subject: [PATCH 040/118] fix(pl-edit): procedure name in drop procedure statements are recognized as the table name in DBSchemaExtractor (#3894) * fix pl name are recognized as the table name * fix typo in test_db_schema_extractor.yaml comment * Add drop stmt handling in DBSchemaExtractor and tests * drop view statement should be extracted --- .../session/util/DBSchemaExtractor.java | 96 +++++++++++++++++++ .../session/test_db_schema_extractor.yaml | 94 +++++++++++++++++- 2 files changed, 189 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java index 03186e1f7a..efb199298a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java @@ -200,6 +200,39 @@ private static class OBMySQLRelationFactorVisitor extends OBParserBaseVisitor identities = new HashSet<>(); + @Override + public RelationFactor visitDrop_function_stmt( + com.oceanbase.tools.sqlparser.obmysql.OBParser.Drop_function_stmtContext ctx) { + RelationFactor relationFactor = MySQLFromReferenceFactory.getRelationFactor( + ctx.relation_factor().normal_relation_factor()); + if (relationFactor.getSchema() != null) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } + return null; + } + + @Override + public RelationFactor visitDrop_procedure_stmt( + com.oceanbase.tools.sqlparser.obmysql.OBParser.Drop_procedure_stmtContext ctx) { + RelationFactor relationFactor = MySQLFromReferenceFactory.getRelationFactor( + ctx.relation_factor().normal_relation_factor()); + if (relationFactor.getSchema() != null) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } + return null; + } + + @Override + public RelationFactor visitDrop_trigger_stmt( + com.oceanbase.tools.sqlparser.obmysql.OBParser.Drop_trigger_stmtContext ctx) { + RelationFactor relationFactor = MySQLFromReferenceFactory.getRelationFactor( + ctx.relation_factor().normal_relation_factor()); + if (relationFactor.getSchema() != null) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } + return null; + } + @Override public RelationFactor visitPartition_option(Partition_optionContext ctx) { return null; @@ -353,6 +386,69 @@ private static class OBOracleRelationFactorVisitor private final Set identities = new HashSet<>(); + @Override + public RelationFactor visitDrop_package_stmt(OBParser.Drop_package_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + if (relationFactor.getSchema() != null) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } + return null; + } + + @Override + public RelationFactor visitDrop_procedure_stmt(OBParser.Drop_procedure_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + if (relationFactor.getSchema() != null) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } + return null; + } + + @Override + public RelationFactor visitDrop_function_stmt(OBParser.Drop_function_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + if (relationFactor.getSchema() != null) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } + return null; + } + + @Override + public RelationFactor visitDrop_trigger_stmt(OBParser.Drop_trigger_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + if (relationFactor.getSchema() != null) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } + return null; + } + + @Override + public RelationFactor visitDrop_type_stmt(OBParser.Drop_type_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + if (relationFactor.getSchema() != null) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } + return null; + } + + @Override + public RelationFactor visitDrop_sequence_stmt(OBParser.Drop_sequence_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + if (relationFactor.getSchema() != null) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } + return null; + } + + @Override + public RelationFactor visitDrop_synonym_stmt(OBParser.Drop_synonym_stmtContext ctx) { + String databaseName = ctx.database_factor().getText(); + if (databaseName != null) { + identities.add(new DBSchemaIdentity(databaseName, null)); + } + return null; + } + @Override public RelationFactor visitPartition_option(OBParser.Partition_optionContext ctx) { return null; diff --git a/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml b/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml index 85e0f90885..e41c1d257a 100644 --- a/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml +++ b/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml @@ -183,4 +183,96 @@ default_schema: DEFAULT_SCHEMA sqls: - "SELECT * FROM DB1.TABLE1@FAKE_DBLINK;" - expected: [ ] \ No newline at end of file + expected: [ ] +# drop function ,procedure and trigger +- id: 23 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop function db1.func1" + expected: + - schema: db1 + table: ~ +- id: 24 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop procedure db1.proc1" + expected: + - schema: db1 + table: ~ +- id: 25 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop trigger db1.trigger1" + expected: + - schema: db1 + table: ~ +- id: 26 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP FUNCTION DB1.FUNC1" + expected: + - schema: DB1 + table: ~ +- id: 27 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PROCEDURE DB1.PROC1" + expected: + - schema: DB1 + table: ~ +- id: 28 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP TRIGGER DB1.TRIGGER1" + expected: + - schema: DB1 + table: ~ +#drop sequence、type、package、synonym、public synonym for ob oracle +- id: 29 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP SEQUENCE DB1.SEQ1" + expected: + - schema: DB1 + table: ~ +- id: 30 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP TYPE DB1.TYPE1" + expected: + - schema: DB1 + table: ~ +- id: 31 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PACKAGE DB1.PACKAGE1" + expected: + - schema: DB1 + table: ~ +- id: 32 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP SYNONYM DB1.SYNONYM1" + expected: + - schema: DB1 + table: ~ +- id : 33 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PUBLIC SYNONYM DB1.SYNONYM1" + expected: + - schema: DB1 + table: ~ + + From 33544cdc334b28b5c7a148b34ad4e9d4fa8e97ef Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Thu, 28 Nov 2024 10:04:11 +0800 Subject: [PATCH 041/118] feat(subpartition): finish sub partitions (#3905) * finish subpartitions for ob mysql * modify according to comments * Enhance OBMySQLGetDBTableByParser for subpartition support * Refactor OBMySQLGetDBTableByParser for subpartition naming clarity * modify format * Enhance OBOracleGetDBTableByParser for subpartition support * Refactor fillSubPartitionValue method * Add tests for ob oracle * Refactor fillSubPartitionValue method for clarity --- .../parser/OBMySQLGetDBTableByParser.java | 142 +++- .../parser/OBMySQLGetDBTableByParserTest.java | 624 +++++++++++++++++- .../parser/OBOracleGetDBTableByParser.java | 130 +++- .../OBOracleGetDBTableByParserTest.java | 351 +++++++++- .../src/test/resources/parser/drop.sql | 23 +- .../resources/parser/testGetTableByParser.sql | 165 ++++- 6 files changed, 1390 insertions(+), 45 deletions(-) diff --git a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java index d5fe07f718..687b0330ef 100644 --- a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java +++ b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java @@ -47,9 +47,13 @@ import com.oceanbase.tools.sqlparser.statement.createtable.OutOfLineConstraint; import com.oceanbase.tools.sqlparser.statement.createtable.OutOfLineForeignConstraint; import com.oceanbase.tools.sqlparser.statement.createtable.Partition; +import com.oceanbase.tools.sqlparser.statement.createtable.PartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartition; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubListPartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionOption; +import com.oceanbase.tools.sqlparser.statement.createtable.SubRangePartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.TableElement; import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; @@ -196,20 +200,11 @@ public List listIndexes() { @Override public DBTablePartition getPartition() { DBTablePartition partition = new DBTablePartition(); - DBTablePartition subPartition = new DBTablePartition(); - partition.setSubpartition(subPartition); - DBTablePartitionOption partitionOption = new DBTablePartitionOption(); partitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); partition.setPartitionOption(partitionOption); - DBTablePartitionOption subPartitionOption = new DBTablePartitionOption(); - subPartitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); - subPartition.setPartitionOption(subPartitionOption); - List partitionDefinitions = new ArrayList<>(); partition.setPartitionDefinitions(partitionDefinitions); - List subPartitionDefinitions = new ArrayList<>(); - subPartition.setPartitionDefinitions(subPartitionDefinitions); if (Objects.isNull(createTableStmt)) { partition.setWarning("Failed to parse table ddl"); @@ -241,36 +236,127 @@ public DBTablePartition getPartition() { partition.getPartitionOption().setExpression(String.join(", ", columnNames)); } } + fillSubPartitions(partition, partitionStmt); + return partition; + } + + private void fillSubPartitions(DBTablePartition partition, Partition partitionStmt) { if (partitionStmt.getSubPartitionOption() == null) { - return partition; + return; } - // TODO 目前 ODC 仅支持 HASH/KEY 二级分区, 其它类型后续需补充 + DBTablePartition subPartition = new DBTablePartition(); + partition.setSubpartition(subPartition); + DBTablePartitionOption subPartitionOption = new DBTablePartitionOption(); + subPartitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); + subPartition.setPartitionOption(subPartitionOption); + List subPartitionDefinitions = new ArrayList<>(); + subPartition.setPartitionDefinitions(subPartitionDefinitions); partition.setSubpartitionTemplated(partitionStmt.getSubPartitionOption().getTemplates() != null); - SubPartitionOption subOption = partitionStmt.getSubPartitionOption(); String type = partitionStmt.getSubPartitionOption().getType(); - if ("key".equals(type.toLowerCase())) { - subPartitionOption.setType(DBTablePartitionType.KEY); - subPartitionOption.setColumnNames(subOption.getSubPartitionTargets() == null ? null - : subOption.getSubPartitionTargets().stream().map(item -> removeIdentifiers(item.getText())) - .collect(Collectors.toList())); - subPartitionOption - .setPartitionsNum(partition.getSubpartitionTemplated() ? subOption.getTemplates().size() - : partitionStmt.getPartitionElements().get(0).getSubPartitionElements().size()); - } else if ("hash".equals(type.toLowerCase())) { - subPartitionOption.setType(DBTablePartitionType.HASH); - Expression expression = subOption.getSubPartitionTargets().get(0); + DBTablePartitionType subDBTablePartitionType = DBTablePartitionType.fromValue(type); + if (DBTablePartitionType.NOT_PARTITIONED == subDBTablePartitionType) { + partition.setWarning("not support this subpartition type, type: " + type); + return; + } + subPartitionOption.setType(subDBTablePartitionType); + SubPartitionOption parsedSubPartitionOption = partitionStmt.getSubPartitionOption(); + // When expressions are supported, only single partition keys are supported + if (subDBTablePartitionType.supportExpression()) { + Expression expression = parsedSubPartitionOption.getSubPartitionTargets().get(0); if (expression instanceof ColumnReference) { subPartitionOption.setColumnNames(Collections.singletonList(removeIdentifiers(expression.getText()))); } else { subPartitionOption.setExpression(expression.getText()); } - subPartitionOption - .setPartitionsNum(partition.getSubpartitionTemplated() ? subOption.getTemplates().size() - : partitionStmt.getPartitionElements().get(0).getSubPartitionElements().size()); } else { - partition.setWarning("Only support HASH/KEY subpartition currently"); + // When expressions are not supported, multiple columns are supported as partition keys + subPartitionOption.setColumnNames(parsedSubPartitionOption.getSubPartitionTargets() == null ? null + : parsedSubPartitionOption.getSubPartitionTargets().stream() + .map(item -> removeIdentifiers(item.getText())) + .collect(Collectors.toList())); + } + /** + * 

+         * subpartitionsNum indicates the number of subpartitions in each partition.
+         * Therefore, subpartitionsNum should be configured only when the subpartitions are templated.
+         * If subpartition is not templated, the subpartitionsNum is not fixed.
+         * such as the following example of non-template subpartition table
+         *
+         * CREATE TABLE ranges_list (col1 INT,col2 INT)
+         *        PARTITION BY RANGE COLUMNS(col1)
+         *        SUBPARTITION BY LIST(col2)
+         *        (PARTITION p0 VALUES LESS THAN(100)
+         *          (SUBPARTITION sp0 VALUES IN(1,3),
+         *           SUBPARTITION sp1 VALUES IN(4,6),
+         *           SUBPARTITION sp2 VALUES IN(7,9)),
+         *         PARTITION p1 VALUES LESS THAN(200)
+         *          (SUBPARTITION sp3 VALUES IN(1,3))
+         *        );
+         *
+         * In this case, the subpartitionsNum is not fixed.
+         *
+ */ + subPartitionOption + .setPartitionsNum( + parsedSubPartitionOption.getTemplates() != null ? parsedSubPartitionOption.getTemplates().size() + : null); + for (PartitionElement partitionElement : partitionStmt.getPartitionElements()) { + if (partitionElement.getSubPartitionElements() != null) { + // obtain DBTablePartitionDefinitions for non-templated subpartitions + for (int i = 0; i < partitionElement.getSubPartitionElements().size(); i++) { + DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); + SubPartitionElement subPartitionElement = partitionElement.getSubPartitionElements().get(i); + fillSubPartitionValue(subPartitionElement, subPartitionDefinition); + subPartitionDefinition.setName( + removeIdentifiers(subPartitionElement.getRelation())); + subPartitionDefinition.setOrdinalPosition(i); + subPartitionDefinition.setType(subDBTablePartitionType); + subPartitionDefinitions.add(subPartitionDefinition); + } + } else { + // obtain DBTablePartitionDefinitions for templated subpartitions + String parentPartitionName = removeIdentifiers(partitionElement.getRelation()); + List templates = partitionStmt.getSubPartitionOption().getTemplates(); + for (int i = 0; i < templates.size(); i++) { + DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); + SubPartitionElement subPartitionElement = templates.get(i); + fillSubPartitionValue(subPartitionElement, subPartitionDefinition); + // for a templated subpartition table, the naming rule for the subpartition is + // '($part_name)s($subpart_name)'. + subPartitionDefinition.setName( + parentPartitionName + 's' + removeIdentifiers(subPartitionElement.getRelation())); + subPartitionDefinition.setOrdinalPosition(i); + subPartitionDefinition.setType(subDBTablePartitionType); + subPartitionDefinitions.add(subPartitionDefinition); + } + } + } + } + + private void fillSubPartitionValue(SubPartitionElement subPartitionElement, + DBTablePartitionDefinition subPartitionDefinition) { + if (subPartitionElement instanceof SubListPartitionElement) { + SubListPartitionElement subListPartitionElement = + (SubListPartitionElement) subPartitionElement; + List> valuesList = new ArrayList<>(); + for (Expression listExpr : subListPartitionElement.getListExprs()) { + if (listExpr instanceof CollectionExpression) { + valuesList.add( + ((CollectionExpression) listExpr).getExpressionList().stream() + .map(Expression::getText) + .collect(Collectors.toList())); + } else if (listExpr instanceof ConstExpression) { + valuesList.add(Collections.singletonList(listExpr.getText())); + } + } + subPartitionDefinition.setValuesList(valuesList); + } else if (subPartitionElement instanceof SubRangePartitionElement) { + SubRangePartitionElement subRangePartitionElement = + (SubRangePartitionElement) subPartitionElement; + subPartitionDefinition.setMaxValues( + subRangePartitionElement.getRangeExprs().stream().map(Expression::getText) + .collect(Collectors.toList())); } - return partition; } private void parseHashPartitionStmt(HashPartition statement, DBTablePartition partition) { diff --git a/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java b/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java index fb460606f2..3a407f20c7 100644 --- a/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java +++ b/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java @@ -230,7 +230,7 @@ public void getPartition_secondary_key_no_template_Success() { Assert.assertEquals("col3", partition.getSubpartition().getPartitionOption().getColumnNames().get(1)); Assert.assertEquals(DBTablePartitionType.KEY, partition.getSubpartition().getPartitionOption().getType()); Assert.assertEquals(2, partition.getSubpartition().getPartitionOption().getColumnNames().size()); - Assert.assertTrue(partition.getSubpartition().getPartitionOption().getPartitionsNum() == 3); + Assert.assertNull(partition.getSubpartition().getPartitionOption().getPartitionsNum()); } @Test @@ -278,7 +278,7 @@ public void getPartition_secondary_hash_no_template_Success() { Assert.assertEquals(false, partition.getSubpartitionTemplated()); Assert.assertEquals(DBTablePartitionType.HASH, partition.getSubpartition().getPartitionOption().getType()); Assert.assertEquals("col2", partition.getSubpartition().getPartitionOption().getColumnNames().get(0)); - Assert.assertTrue(partition.getSubpartition().getPartitionOption().getPartitionsNum() == 3); + Assert.assertNull(partition.getSubpartition().getPartitionOption().getPartitionsNum()); } @Test @@ -345,4 +345,624 @@ public void getConstraints_test_out_line_constraints_with_idx_name_Success() { Assert.assertEquals(constraints.get(0).getName(), "idx_name"); Assert.assertEquals(constraints.get(0).getType(), DBConstraintType.UNIQUE_KEY); } + + @Test + public void getSubpartition_RangeColumnsAndRange_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_range (col1 INT NOT NULL,col2 varchar(50),col3 INT NOT NULL) \n" + + "PARTITION BY RANGE COLUMNS(col1)\n" + + "SUBPARTITION BY RANGE(col3)\n" + + "SUBPARTITION TEMPLATE \n" + + "(SUBPARTITION mp0 VALUES LESS THAN(1000),\n" + + " SUBPARTITION mp1 VALUES LESS THAN(2000),\n" + + " SUBPARTITION mp2 VALUES LESS THAN(3000)\n" + + ")\n" + + "(PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + "); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + } + + @Test + public void getSubpartition_RangeColumnsAndRange_ExpressionKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_range_expr (col1 INT NOT NULL,col2 varchar(50),col3 INT NOT NULL) \n" + + "PARTITION BY RANGE COLUMNS(col1)\n" + + "SUBPARTITION BY RANGE(ABS(col3))\n" + + "SUBPARTITION TEMPLATE \n" + + "(SUBPARTITION mp0 VALUES LESS THAN(1000),\n" + + " SUBPARTITION mp1 VALUES LESS THAN(2000),\n" + + " SUBPARTITION mp2 VALUES LESS THAN(3000)\n" + + ")\n" + + "(PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + "); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals("ABS(col3)", subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + } + + @Test + public void getSubpartition_RangeAndRange_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE range_range(col1 INT,col2 INT) \n" + + " PARTITION BY RANGE(col1)\n" + + " SUBPARTITION BY RANGE(col2)\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES LESS THAN(100),\n" + + " SUBPARTITION sp1 VALUES LESS THAN(200),\n" + + " SUBPARTITION sp2 VALUES LESS THAN(300),\n" + + " SUBPARTITION sp3 VALUES LESS THAN(400)\n" + + " ),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp4 VALUES LESS THAN(100),\n" + + " SUBPARTITION sp5 VALUES LESS THAN(200),\n" + + " SUBPARTITION sp6 VALUES LESS THAN(300),\n" + + " SUBPARTITION sp7 VALUES LESS THAN(400)\n" + + " )\n" + + " );\n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + } + + @Test + public void getSubpartition_RangeAndRange_ExpressionKey_NoTemplate_Success() { + String ddl = "CREATE TABLE range_range_expr(col1 INT,col2 TIMESTAMP) \n" + + " PARTITION BY RANGE(col1)\n" + + " SUBPARTITION BY RANGE(UNIX_TIMESTAMP(col2))\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES LESS THAN(UNIX_TIMESTAMP('2021/04/01')),\n" + + " SUBPARTITION sp1 VALUES LESS THAN(UNIX_TIMESTAMP('2021/07/01')),\n" + + " SUBPARTITION sp2 VALUES LESS THAN(UNIX_TIMESTAMP('2021/10/01')),\n" + + " SUBPARTITION sp3 VALUES LESS THAN(UNIX_TIMESTAMP('2022/01/01'))\n" + + " ),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp4 VALUES LESS THAN(UNIX_TIMESTAMP('2021/04/01')),\n" + + " SUBPARTITION sp5 VALUES LESS THAN(UNIX_TIMESTAMP('2021/07/01')),\n" + + " SUBPARTITION sp6 VALUES LESS THAN(UNIX_TIMESTAMP('2021/10/01')),\n" + + " SUBPARTITION sp7 VALUES LESS THAN(UNIX_TIMESTAMP('2022/01/01'))\n" + + " )\n" + + " );\n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals("UNIX_TIMESTAMP(col2)", subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("UNIX_TIMESTAMP('2021/04/01')", + subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + } + + @Test + public void getSubpartition_RangeColumnsAndRangeColumns_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_ranges(col1 INT,col2 INT,col3 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY RANGE COLUMNS(col2,col3)\n" + + " SUBPARTITION TEMPLATE \n" + + " (SUBPARTITION mp0 VALUES LESS THAN(1000,1000),\n" + + " SUBPARTITION mp1 VALUES LESS THAN(2000,2000),\n" + + " SUBPARTITION mp2 VALUES LESS THAN(3000,3000)\n" + + " )\n" + + " (PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + " ); "; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + } + + @Test + public void getSubpartition_RangeColumnsAndRangeColumns_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_ranges (col1 INT NOT NULL,col2 INT NOT NULL,col3 INT NOT NULL) \n" + + "PARTITION BY RANGE COLUMNS(col1)\n" + + "SUBPARTITION BY RANGE COLUMNS(col2,col3)\n" + + "(PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES LESS THAN(1000,1000),\n" + + " SUBPARTITION sp1 VALUES LESS THAN(2000,2000),\n" + + " SUBPARTITION sp2 VALUES LESS THAN(3000,3000)),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3 VALUES LESS THAN(1000,1000),\n" + + " SUBPARTITION sp4 VALUES LESS THAN(2000,2000),\n" + + " SUBPARTITION sp5 VALUES LESS THAN(3000,3000))\n" + + ");\n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + } + + @Test + public void getSubpartition_RangeColumnsAndList_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_list(col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST(col2)\n" + + " SUBPARTITION TEMPLATE \n" + + " (SUBPARTITION mp0 VALUES IN(1,3),\n" + + " SUBPARTITION mp1 VALUES IN(4,6),\n" + + " SUBPARTITION mp2 VALUES IN(7)\n" + + " )\n" + + " (PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + } + + @Test + public void getSubpartition_RangeColumnsAndList_ExpressionKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_list_expr(col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST(abs(col2))\n" + + " SUBPARTITION TEMPLATE \n" + + " (SUBPARTITION mp0 VALUES IN(1,3),\n" + + " SUBPARTITION mp1 VALUES IN(4,6),\n" + + " SUBPARTITION mp2 VALUES IN(7)\n" + + " )\n" + + " (PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals("abs(col2)", subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + } + + @Test + public void getSubpartition_RangeColumnsAndList_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_list (col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST(col2)\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES IN(1,3),\n" + + " SUBPARTITION sp1 VALUES IN(4,6),\n" + + " SUBPARTITION sp2 VALUES IN(7,9)),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3 VALUES IN(1,3))\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + } + + @Test + public void getSubpartition_RangeColumnsAndList_ExpressionKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_list_expr (col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST(abs(col2))\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES IN(1,3),\n" + + " SUBPARTITION sp1 VALUES IN(4,6),\n" + + " SUBPARTITION sp2 VALUES IN(7,9)),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3 VALUES IN(1,3))\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals("abs(col2)", subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + } + + @Test + public void getSubpartition_RangeColumnsAndListColumns_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_lists(col1 INT,col2 INT,col3 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST COLUMNS(col2,col3)\n" + + " SUBPARTITION TEMPLATE \n" + + " (SUBPARTITION mp0 VALUES IN((1,1),(3,3)),\n" + + " SUBPARTITION mp1 VALUES IN((4,4),(6,6)),\n" + + " SUBPARTITION mp2 VALUES IN((7,7),(8,8))\n" + + " )\n" + + " (PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST_COLUMNS, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(1)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(1)); + } + + @Test + public void getSubpartition_RangeColumnsAndListColumns_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_lists (col1 INT,col2 INT,col3 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST COLUMNS(col2,col3)\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES IN(1,1),\n" + + " SUBPARTITION sp1 VALUES IN(4,4),\n" + + " SUBPARTITION sp2 VALUES IN(7,7)),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3 VALUES IN(9,9),\n" + + " SUBPARTITION sp4 VALUES IN(10,10),\n" + + " SUBPARTITION sp5 VALUES IN(11,11))\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST_COLUMNS, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + } + + @Test + public void getSubpartition_RangeColumnsAndHash_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE `t_ranges_hash` (\n" + + " `col1` int(11) DEFAULT NULL,\n" + + " `col2` int(11) DEFAULT NULL\n" + + ") \n" + + " partition by range columns(col1) subpartition by hash(col2) subpartition template (\n" + + "subpartition `p0`,\n" + + "subpartition `p1`,\n" + + "subpartition `p2`,\n" + + "subpartition `p3`,\n" + + "subpartition `p4`)\n" + + "(partition `p0` values less than (100),\n" + + "partition `p1` values less than (200),\n" + + "partition `p2` values less than (300))"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(5L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0sp0", subpartition.getPartitionDefinitions().get(0).getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndHash_ExpressionKey_Template_Success() { + String ddl = "CREATE TABLE `t_ranges_hash_expr` (\n" + + " `col1` int(11) DEFAULT NULL,\n" + + " `col2` int(11) DEFAULT NULL\n" + + ") \n" + + " partition by range columns(col1) subpartition by hash(abs(col2)) subpartition template (\n" + + "subpartition `p0`,\n" + + "subpartition `p1`,\n" + + "subpartition `p2`,\n" + + "subpartition `p3`,\n" + + "subpartition `p4`)\n" + + "(partition `p0` values less than (100),\n" + + "partition `p1` values less than (200),\n" + + "partition `p2` values less than (300))"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals("abs(col2)", subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(5L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0sp0", subpartition.getPartitionDefinitions().get(0).getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndHash_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_hash (col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY HASH(col2)\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0,\n" + + " SUBPARTITION sp1,\n" + + " SUBPARTITION sp2),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3,\n" + + " SUBPARTITION sp4,\n" + + " SUBPARTITION sp5)\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndHash_ExpressionKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_hash_expr (col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY HASH(abs(col2))\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0,\n" + + " SUBPARTITION sp1,\n" + + " SUBPARTITION sp2),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3,\n" + + " SUBPARTITION sp4,\n" + + " SUBPARTITION sp5)\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals("abs(col2)", subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndKey_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE `t_ranges_key` (\n" + + " `col1` int(11) DEFAULT NULL,\n" + + " `col2` int(11) DEFAULT NULL\n" + + ") \n" + + " partition by range columns(col1) subpartition by key(col2) subpartition template (\n" + + "subpartition `p0`,\n" + + "subpartition `p1`,\n" + + "subpartition `p2`)\n" + + "(partition `p0` values less than (100),\n" + + "partition `p1` values less than (200),\n" + + "partition `p2` values less than (300))"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.KEY, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0sp0", subpartition.getPartitionDefinitions().get(0).getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndKey_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE `ranges_key` (\n" + + " `col1` int(11) NOT NULL,\n" + + " `col2` varchar(50) DEFAULT NULL,\n" + + " `col3` int(11) NOT NULL\n" + + ") \n" + + " partition by range columns(col1) subpartition by key(col3)\n" + + "(partition `p0` values less than (100) (\n" + + "subpartition `sp0`,\n" + + "subpartition `sp1`,\n" + + "subpartition `sp2`),\n" + + "partition `p1` values less than (200) (\n" + + "subpartition `sp3`,\n" + + "subpartition `sp4`))"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.KEY, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + } + } diff --git a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java index dab498c8fd..98bf8a51a9 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java +++ b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java @@ -60,9 +60,16 @@ import com.oceanbase.tools.sqlparser.statement.createtable.OutOfLineConstraint; import com.oceanbase.tools.sqlparser.statement.createtable.OutOfLineForeignConstraint; import com.oceanbase.tools.sqlparser.statement.createtable.Partition; +import com.oceanbase.tools.sqlparser.statement.createtable.PartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartition; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubListPartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionOption; +import com.oceanbase.tools.sqlparser.statement.createtable.SubRangePartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.TableElement; +import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; import com.oceanbase.tools.sqlparser.statement.expression.RelationReference; import lombok.Getter; @@ -405,9 +412,121 @@ public DBTablePartition getPartition() { partition.getPartitionOption().setExpression(String.join(", ", columnNames)); } } + fillSubPartitions(partition, partitionStmt); return partition; } + private void fillSubPartitions(DBTablePartition partition, Partition partitionStmt) { + if (partitionStmt.getSubPartitionOption() == null) { + return; + } + DBTablePartition subPartition = new DBTablePartition(); + partition.setSubpartition(subPartition); + DBTablePartitionOption subPartitionOption = new DBTablePartitionOption(); + subPartitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); + subPartition.setPartitionOption(subPartitionOption); + List subPartitionDefinitions = new ArrayList<>(); + subPartition.setPartitionDefinitions(subPartitionDefinitions); + partition.setSubpartitionTemplated(partitionStmt.getSubPartitionOption().getTemplates() != null); + String type = partitionStmt.getSubPartitionOption().getType(); + DBTablePartitionType subDBTablePartitionType = DBTablePartitionType.fromValue(type); + if (DBTablePartitionType.NOT_PARTITIONED == subDBTablePartitionType) { + partition.setWarning("not support this subpartition type, type: " + type); + return; + } + subPartitionOption.setType(subDBTablePartitionType); + SubPartitionOption parsedSubPartitionOption = partitionStmt.getSubPartitionOption(); + // expressions are not supported, multiple columns are supported as partition keys + subPartitionOption.setColumnNames(parsedSubPartitionOption.getSubPartitionTargets() == null ? null + : parsedSubPartitionOption.getSubPartitionTargets().stream() + .map(item -> removeIdentifiers(item.getText())) + .collect(Collectors.toList())); + + /** + * 
+         * subpartitionsNum indicates the number of subpartitions in each partition.
+         * Therefore, subpartitionsNum should be configured only when the subpartitions are templated.
+         * If subpartition is not templated, the subpartitionsNum is not fixed.
+         * such as the following example of non-template subpartition table
+         *
+         * CREATE TABLE range_range(col1 INT,col2 INT,col3 INT)
+         *        PARTITION BY RANGE(col1)
+         *        SUBPARTITION BY RANGE(col2,col3)
+         *         (PARTITION p0 VALUES LESS THAN(100)
+         *           (SUBPARTITION sp0 VALUES LESS THAN(2020,2020),
+         *            SUBPARTITION sp1 VALUES LESS THAN(2021,2021)
+         *           ),
+         *          PARTITION p1 VALUES LESS THAN(200)
+         *           (SUBPARTITION sp2 VALUES LESS THAN(2020,2020),
+         *            SUBPARTITION sp3 VALUES LESS THAN(2021,2021),
+         *            SUBPARTITION sp4 VALUES LESS THAN(2022,2022)
+         *            )
+         *          );
+         *
+         * In this case, the subpartitionsNum is not fixed.
+         *
+ */ + subPartitionOption + .setPartitionsNum( + parsedSubPartitionOption.getTemplates() != null ? parsedSubPartitionOption.getTemplates().size() + : null); + for (PartitionElement partitionElement : partitionStmt.getPartitionElements()) { + if (partitionElement.getSubPartitionElements() != null) { + // obtain DBTablePartitionDefinitions for non-templated subpartitions + for (int i = 0; i < partitionElement.getSubPartitionElements().size(); i++) { + DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); + SubPartitionElement subPartitionElement = partitionElement.getSubPartitionElements().get(i); + fillSubPartitionValue(subPartitionElement, subPartitionDefinition); + subPartitionDefinition.setName( + removeIdentifiers(subPartitionElement.getRelation())); + subPartitionDefinition.setOrdinalPosition(i); + subPartitionDefinition.setType(subDBTablePartitionType); + subPartitionDefinitions.add(subPartitionDefinition); + } + } else { + // obtain DBTablePartitionDefinitions for templated subpartitions + String parentPartitionName = removeIdentifiers(partitionElement.getRelation()); + List templates = partitionStmt.getSubPartitionOption().getTemplates(); + for (int i = 0; i < templates.size(); i++) { + DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); + SubPartitionElement subPartitionElement = templates.get(i); + fillSubPartitionValue(subPartitionElement, subPartitionDefinition); + // for a templated subpartition table, the naming rule for the subpartition is + // '($part_name)S($subpart_name)'. + subPartitionDefinition.setName( + parentPartitionName + 'S' + removeIdentifiers(subPartitionElement.getRelation())); + subPartitionDefinition.setOrdinalPosition(i); + subPartitionDefinition.setType(subDBTablePartitionType); + subPartitionDefinitions.add(subPartitionDefinition); + } + } + } + } + + private void fillSubPartitionValue(SubPartitionElement subPartitionElement, + DBTablePartitionDefinition subPartitionDefinition) { + if (subPartitionElement instanceof SubRangePartitionElement) { + SubRangePartitionElement subRangePartitionElement = (SubRangePartitionElement) subPartitionElement; + subPartitionDefinition.setMaxValues( + subRangePartitionElement.getRangeExprs().stream().map(Expression::getText) + .collect(Collectors.toList())); + } else if (subPartitionElement instanceof SubListPartitionElement) { + SubListPartitionElement subListPartitionElement = (SubListPartitionElement) subPartitionElement; + List> valuesList = new ArrayList<>(); + for (Expression listExpr : subListPartitionElement.getListExprs()) { + if (listExpr instanceof CollectionExpression) { + valuesList.add( + ((CollectionExpression) listExpr).getExpressionList().stream() + .map(Expression::getText) + .collect(Collectors.toList())); + } else if (listExpr instanceof ConstExpression) { + valuesList.add(Collections.singletonList(listExpr.getText())); + } + } + subPartitionDefinition.setValuesList(valuesList); + } + } + private void parseHashPartitionStmt(HashPartition statement, DBTablePartition partition) { DBTablePartitionOption option = partition.getPartitionOption(); option.setType(DBTablePartitionType.HASH); @@ -470,7 +589,16 @@ private void parseListPartitionStmt(ListPartition statement, DBTablePartition pa DBTablePartitionDefinition partitionDefinition = new DBTablePartitionDefinition(); partitionDefinition.setOrdinalPosition(i); List> valuesList = new ArrayList<>(); - element.getListExprs().forEach(item -> valuesList.add(Collections.singletonList(item.getText()))); + for (Expression listExpr : element.getListExprs()) { + if (listExpr instanceof CollectionExpression) { + valuesList.add( + ((CollectionExpression) listExpr).getExpressionList().stream() + .map(Expression::getText) + .collect(Collectors.toList())); + } else if (listExpr instanceof ConstExpression) { + valuesList.add(Collections.singletonList(listExpr.getText())); + } + } partitionDefinition.setValuesList(valuesList); partitionDefinition.setName(removeIdentifiers(element.getRelation())); partitionDefinition.setType(DBTablePartitionType.LIST); diff --git a/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java b/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java index e87a902f5d..dbfe201610 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java +++ b/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java @@ -69,33 +69,372 @@ private static void batchExcuteSql(String str) { } @Test - public void getPartition_Hash_Success() { + public void getPartition_Hash_SingleKey_Success() { OBOracleGetDBTableByParser table = - new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "HASH_PART_BY_PARSER"); + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "HASH_PART_SINGLE_KEY_BY_PARSER"); DBTablePartition partition = table.getPartition(); Assert.assertEquals(6L, partition.getPartitionOption().getPartitionsNum().longValue()); Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); } @Test - public void getPartition_list_Success() { + public void getPartition_Hash_MultipleKey_Success() { OBOracleGetDBTableByParser table = - new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "LIST_PART_BY_PARSER"); + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "HASH_PART_MULTIPLE_KEY_BY_PARSER"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(10L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(2, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL2", partition.getPartitionOption().getColumnNames().get(1)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); + } + + @Test + public void getPartition_List_SingleKey_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "LIST_PART_SINGLE_KEY_BY_PARSER"); DBTablePartition partition = table.getPartition(); Assert.assertEquals(4L, partition.getPartitionOption().getPartitionsNum().longValue()); Assert.assertEquals(DBTablePartitionType.LIST, partition.getPartitionOption().getType()); Assert.assertEquals("LOG_VALUE", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P01", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(3, partition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("'A'", partition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); } @Test - public void getPartition_range_Success() { + public void getPartition_List_MultipleKey_Success() { OBOracleGetDBTableByParser table = - new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "RANGE_PART_BY_PARSER"); + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "LIST_PART_MULTIPLE_KEY_BY_PARSER"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(3L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.LIST, partition.getPartitionOption().getType()); + Assert.assertEquals(2, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL2", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, partition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals(2, partition.getPartitionDefinitions().get(0).getValuesList().get(0).size()); + Assert.assertEquals("1", partition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("1", partition.getPartitionDefinitions().get(0).getValuesList().get(0).get(1)); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); + } + + @Test + public void getPartition_Range_SingleKey_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "RANGE_PART_SINGLE_KEY_BY_PARSER"); DBTablePartition partition = table.getPartition(); Assert.assertEquals(4L, partition.getPartitionOption().getPartitionsNum().longValue()); Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); Assert.assertEquals("LOG_DATE", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("M202001", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("TO_DATE(' 2020-02-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')", + partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); + } + + @Test + public void getPartition_Range_MultipleKey_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "RANGE_PART_MULTIPLE_KEY_BY_PARSER"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(4L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(2, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL2", partition.getPartitionOption().getColumnNames().get(1)); + Assert.assertEquals("M202001", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("1", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("1", partition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); + } + + @Test + public void getSubPartition_TemplateSingleRangeMultipleRange_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_RANGE_MULTIPLE_RANGE"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SMP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + } + + @Test + public void getSubPartition_TemplateSingleRangeSingleRange_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_RANGE_SINGLE_RANGE"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SMP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + } + + @Test + public void getSubPartition_TemplateSingleHashMultipleList_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_HASH_MULTIPLE_LIST"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(5, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SSP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(1)); + } + + @Test + public void getSubPartition_TemplateSingleHashSingleList_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_HASH_SINGLE_LIST"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(5, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SSP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + } + + @Test + public void getSubPartition_TemplateSingleHashMultipleHash_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_HASH_MULTIPLE_HASH"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(5, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SP0", subpartition.getPartitionDefinitions().get(0).getName()); + } + + @Test + public void getSubPartition_TemplateSingleHashSingleHash_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_HASH_SINGLE_HASH"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(5, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SP0", subpartition.getPartitionDefinitions().get(0).getName()); + } + + @Test + public void getSubPartition_SingleRangeMultipleRange_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_RANGE_MULTIPLE_RANGE"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + } + + @Test + public void getSubPartition_SingleRangeSingleRange_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_RANGE_SINGLE_RANGE"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + } + + @Test + public void getSubPartition_SingleHashMultipleList_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_HASH_MULTIPLE_LIST"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(1)); + } + + @Test + public void getSubPartition_SingleHashSingleList_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_HASH_SINGLE_LIST"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + } + + @Test + public void getSubPartition_SingleListMultipleHash_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_LIST_SINGLE_HASH"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.LIST, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + } + + @Test + public void getSubPartition_SingleListSingleHash_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_LIST_SINGLE_HASH"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.LIST, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); } @Test diff --git a/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/drop.sql b/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/drop.sql index be9424c10b..d2771f7f0f 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/drop.sql +++ b/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/drop.sql @@ -13,9 +13,24 @@ BEGIN END; / -call DROPIFEXISTS_TABLE('HASH_PART_BY_PARSER'); -call DROPIFEXISTS_TABLE('LIST_PART_BY_PARSER'); -call DROPIFEXISTS_TABLE('RANGE_PART_BY_PARSER'); +call DROPIFEXISTS_TABLE('HASH_PART_SINGLE_KEY_BY_PARSER'); +call DROPIFEXISTS_TABLE('HASH_PART_MULTIPLE_KEY_BY_PARSER'); +call DROPIFEXISTS_TABLE('LIST_PART_SINGLE_KEY_BY_PARSER'); +call DROPIFEXISTS_TABLE('LIST_PART_MULTIPLE_KEY_BY_PARSER'); +call DROPIFEXISTS_TABLE('RANGE_PART_SINGLE_KEY_BY_PARSER'); +call DROPIFEXISTS_TABLE('RANGE_PART_MULTIPLE_KEY_BY_PARSER'); call DROPIFEXISTS_TABLE('CONSTRAINT_MULTY_BY_PARSER'); call DROPIFEXISTS_TABLE('CONSTRAINT_PRIMARY_BY_PARSER'); -call DROPIFEXISTS_TABLE('TEST_INDEX_BY_PARSER'); \ No newline at end of file +call DROPIFEXISTS_TABLE('TEST_INDEX_BY_PARSER'); +call DROPIFEXISTS_TABLE('T_SINGLE_RANGE_MULTIPLE_RANGE'); +call DROPIFEXISTS_TABLE('T_SINGLE_HASH_MULTIPLE_LIST'); +call DROPIFEXISTS_TABLE('T_SINGLE_HASH_MULTIPLE_HASH'); +call DROPIFEXISTS_TABLE('SINGLE_RANGE_MULTIPLE_RANGE'); +call DROPIFEXISTS_TABLE('SINGLE_HASH_MULTIPLE_LIST'); +call DROPIFEXISTS_TABLE('SINGLE_LIST_MULTIPLE_HASH'); +call DROPIFEXISTS_TABLE('T_SINGLE_RANGE_SINGLE_RANGE'); +call DROPIFEXISTS_TABLE('T_SINGLE_HASH_SINGLE_LIST'); +call DROPIFEXISTS_TABLE('T_SINGLE_HASH_SINGLE_HASH'); +call DROPIFEXISTS_TABLE('SINGLE_RANGE_SINGLE_RANGE'); +call DROPIFEXISTS_TABLE('SINGLE_HASH_SINGLE_LIST'); +call DROPIFEXISTS_TABLE('SINGLE_LIST_SINGLE_HASH'); \ No newline at end of file diff --git a/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/testGetTableByParser.sql b/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/testGetTableByParser.sql index eb4ad0cae5..2a39073e71 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/testGetTableByParser.sql +++ b/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/testGetTableByParser.sql @@ -1,4 +1,4 @@ -CREATE TABLE HASH_PART_BY_PARSER (COL1 NUMBER, COL2 VARCHAR2(50)) +CREATE TABLE HASH_PART_SINGLE_KEY_BY_PARSER (COL1 NUMBER, COL2 VARCHAR2(50)) partition by hash(col1) (partition P0, partition P1, @@ -7,14 +7,24 @@ CREATE TABLE HASH_PART_BY_PARSER (COL1 NUMBER, COL2 VARCHAR2(50)) partition P4, partition P5); -CREATE TABLE LIST_PART_BY_PARSER (LOG_ID NUMBER(38), LOG_VALUE VARCHAR2(20)) +CREATE TABLE HASH_PART_MULTIPLE_KEY_BY_PARSER(COL1 INT,COL2 VARCHAR(50)) + PARTITION BY HASH(COL1,COL2) PARTITIONS 10; + +CREATE TABLE LIST_PART_SINGLE_KEY_BY_PARSER (LOG_ID NUMBER(38), LOG_VALUE VARCHAR2(20)) partition by list(log_value) (partition P01 values ('A','B','C'), partition P02 values ('D','E','F'), partition P03 values ('G','H','I'), partition P04 values (DEFAULT)); -CREATE TABLE RANGE_PART_BY_PARSER ( +CREATE TABLE LIST_PART_MULTIPLE_KEY_BY_PARSER(LOG_ID INT,COL2 INT,COL3 INT) + PARTITION BY LIST(COL2,COL3) + (PARTITION "P0" VALUES ((1,1),(3,3)), + PARTITION "P1" VALUES ((4,4),(7,7)), + PARTITION "P2" VALUES ((8,8),(9,9)) + ); + +CREATE TABLE RANGE_PART_SINGLE_KEY_BY_PARSER ( LOG_ID NUMBER(38), LOG_DATE DATE DEFAULT sysdate CONSTRAINT "RANGE_PART_OBNOTNULL_1688526018016703" NOT NULL ENABLE) partition by range(log_date) @@ -23,6 +33,14 @@ CREATE TABLE RANGE_PART_BY_PARSER ( partition M202003 values less than (TO_DATE('2020-04-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=GREGORIAN')), partition MMAX values less than (MAXVALUE)); +CREATE TABLE RANGE_PART_MULTIPLE_KEY_BY_PARSER(LOG_ID INT,COL1 INT,COL2 INT) + PARTITION BY RANGE(COL1,COL2) + (PARTITION M202001 VALUES LESS THAN(1,1) + , PARTITION M202002 VALUES LESS THAN(3,3) + , PARTITION M202003 VALUES LESS THAN(5,5) + , PARTITION MMAX VALUES LESS THAN (MAXVALUE,MAXVALUE) + ); + CREATE TABLE CONSTRAINT_PRIMARY_BY_PARSER ( COL1 NUMBER(38), COL2 NUMBER(38), @@ -65,4 +83,143 @@ CREATE INDEX IND_FUNCTION_BASED on TEST_INDEX_BY_PARSER (UPPER("COL1")) GLOBAL; CREATE UNIQUE INDEX UNIQUE_IDX_TEST_INDEX_BY_PARSER on TEST_INDEX_BY_PARSER (COL3, COL4) LOCAL; CREATE INDEX NORMAL_IDX_TEST_INDEX_BY_PARSER on TEST_INDEX_BY_PARSER (COL7) LOCAL; -CREATE INDEX IND_FUNCTION_BASED2 ON TEST_INDEX_BY_PARSER(COL8+1); \ No newline at end of file +CREATE INDEX IND_FUNCTION_BASED2 ON TEST_INDEX_BY_PARSER(COL8+1); + +CREATE TABLE T_SINGLE_RANGE_MULTIPLE_RANGE(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY RANGE(COL1) + SUBPARTITION BY RANGE(COL2,COL3) + SUBPARTITION TEMPLATE + (SUBPARTITION MP0 VALUES LESS THAN(2020,2020), + SUBPARTITION MP1 VALUES LESS THAN(2021,2021), + SUBPARTITION MP2 VALUES LESS THAN(2022,2022) + ) + (PARTITION P0 VALUES LESS THAN(100), + PARTITION P1 VALUES LESS THAN(200) + ); + +CREATE TABLE T_SINGLE_HASH_MULTIPLE_LIST(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY LIST(COL2,COL3) + SUBPARTITION TEMPLATE + (SUBPARTITION SP0 VALUES(100,100), + SUBPARTITION SP1 VALUES(200,200), + SUBPARTITION SP2 VALUES(300,300) + ) + PARTITIONS 5; + +CREATE TABLE T_SINGLE_HASH_MULTIPLE_HASH(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY HASH(COL2,COL3) + SUBPARTITIONS 3 + PARTITIONS 5; + +CREATE TABLE SINGLE_RANGE_MULTIPLE_RANGE(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY RANGE(COL1) + SUBPARTITION BY RANGE(COL2,COL3) + (PARTITION P0 VALUES LESS THAN(100) + (SUBPARTITION SP0 VALUES LESS THAN(2020,2020), + SUBPARTITION SP1 VALUES LESS THAN(2021,2021) + ), + PARTITION P1 VALUES LESS THAN(200) + (SUBPARTITION SP2 VALUES LESS THAN(2020,2020), + SUBPARTITION SP3 VALUES LESS THAN(2021,2021), + SUBPARTITION SP4 VALUES LESS THAN(2022,2022) + ) + ); + +CREATE TABLE SINGLE_HASH_MULTIPLE_LIST(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY LIST(COL2,COL3) + (PARTITION P0 + (SUBPARTITION SP0 VALUES((1,1),(3,3)), + SUBPARTITION SP1 VALUES((4,4),(7,7)) + ), + PARTITION P1 + (SUBPARTITION SP2 VALUES((1,1),(3,3)), + SUBPARTITION SP3 VALUES((4,4),(7,7)), + SUBPARTITION SP4 VALUES((8,8),(9,9)) + ) + ); + +CREATE TABLE SINGLE_LIST_MULTIPLE_HASH(COL1 INT,COL2 VARCHAR2(50),COL3 VARCHAR2(50)) + PARTITION BY LIST(COL1) + SUBPARTITION BY HASH(COL2,COL3) + (PARTITION P0 VALUES('01') + (SUBPARTITION SP0, + SUBPARTITION SP1 + ), + PARTITION P1 VALUES('02') + (SUBPARTITION SP2, + SUBPARTITION SP3, + SUBPARTITION SP4 + ) + ); + +CREATE TABLE T_SINGLE_RANGE_SINGLE_RANGE(COL1 INT,COL2 INT) + PARTITION BY RANGE(COL1) + SUBPARTITION BY RANGE(COL2) + SUBPARTITION TEMPLATE + (SUBPARTITION MP0 VALUES LESS THAN(2020), + SUBPARTITION MP1 VALUES LESS THAN(2021), + SUBPARTITION MP2 VALUES LESS THAN(2022) + ) + (PARTITION P0 VALUES LESS THAN(100), + PARTITION P1 VALUES LESS THAN(200) + ); + +CREATE TABLE T_SINGLE_HASH_SINGLE_LIST(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY LIST(COL2) + SUBPARTITION TEMPLATE + (SUBPARTITION SP0 VALUES(100), + SUBPARTITION SP1 VALUES(200), + SUBPARTITION SP2 VALUES(300) + ) + PARTITIONS 5; + +CREATE TABLE T_SINGLE_HASH_SINGLE_HASH(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY HASH(COL2) + SUBPARTITIONS 3 + PARTITIONS 5; + +CREATE TABLE SINGLE_RANGE_SINGLE_RANGE(COL1 INT,COL2 INT) + PARTITION BY RANGE(COL1) + SUBPARTITION BY RANGE(COL2) + (PARTITION P0 VALUES LESS THAN(100) + (SUBPARTITION SP0 VALUES LESS THAN(2020), + SUBPARTITION SP1 VALUES LESS THAN(2021) + ), + PARTITION P1 VALUES LESS THAN(200) + (SUBPARTITION SP2 VALUES LESS THAN(2020), + SUBPARTITION SP3 VALUES LESS THAN(2021), + SUBPARTITION SP4 VALUES LESS THAN(2022) + ) + ); + +CREATE TABLE SINGLE_HASH_SINGLE_LIST(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY LIST(COL2) + (PARTITION P0 + (SUBPARTITION SP0 VALUES(1,3), + SUBPARTITION SP1 VALUES(4,7) + ), + PARTITION P1 + (SUBPARTITION SP2 VALUES(1,3), + SUBPARTITION SP3 VALUES(4,7) + ) + ); + +CREATE TABLE SINGLE_LIST_SINGLE_HASH(COL1 INT,COL2 VARCHAR2(50)) + PARTITION BY LIST(COL1) + SUBPARTITION BY HASH(COL2) + (PARTITION P0 VALUES('01') + (SUBPARTITION SP0, + SUBPARTITION SP1 + ), + PARTITION P1 VALUES('02') + (SUBPARTITION SP2, + SUBPARTITION SP3, + SUBPARTITION SP4 + ) + ); From 2ca87c21c490ca33d0ee96c78664a2010d041799 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Mon, 2 Dec 2024 09:59:12 +0800 Subject: [PATCH 042/118] feat(sub partition) : add the associated partition information for sub partition (#3926) * Adds the associated partition information for subpartition * Add @ApiOperation to DBTableController getTable endpoint * Refactor partition parsing in OBMySQL and OBOracle plugins --- .../model/DBTablePartitionDefinition.java | 1 + .../web/controller/v2/DBTableController.java | 1 + .../parser/BaseOBGetDBTableByParser.java | 267 ++++++++++++++++++ .../parser/OBMySQLGetDBTableByParser.java | 147 ++-------- .../parser/OBMySQLGetDBTableByParserTest.java | 36 +++ .../parser/OBOracleGetDBTableByParser.java | 177 ++---------- .../OBOracleGetDBTableByParserTest.java | 24 ++ 7 files changed, 374 insertions(+), 279 deletions(-) create mode 100644 server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/BaseOBGetDBTableByParser.java diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTablePartitionDefinition.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTablePartitionDefinition.java index 75851e560d..6879a7f437 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTablePartitionDefinition.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTablePartitionDefinition.java @@ -19,4 +19,5 @@ @Data public class DBTablePartitionDefinition extends DBTableAbstractPartitionDefinition { + private DBTablePartitionDefinition parentPartitionDefinition; } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java index e2118bd231..90b7d9bb76 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java @@ -72,6 +72,7 @@ public ListResponse listTables(@PathVariable String sessionId, } } + @ApiOperation(value = "getTable", notes = "get table") @GetMapping(value = {"/{sessionId}/databases/{databaseName}/tables/{tableName}", "/{sessionId}/currentDatabase/tables/{tableName}"}) @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sessionId") diff --git a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/BaseOBGetDBTableByParser.java b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/BaseOBGetDBTableByParser.java new file mode 100644 index 0000000000..5eb5814d80 --- /dev/null +++ b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/BaseOBGetDBTableByParser.java @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.plugin.schema.obmysql.parser; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.tools.dbbrowser.model.DBTableColumn; +import com.oceanbase.tools.dbbrowser.model.DBTableIndex; +import com.oceanbase.tools.dbbrowser.model.DBTablePartition; +import com.oceanbase.tools.dbbrowser.model.DBTablePartitionDefinition; +import com.oceanbase.tools.dbbrowser.model.DBTablePartitionOption; +import com.oceanbase.tools.dbbrowser.model.DBTablePartitionType; +import com.oceanbase.tools.sqlparser.statement.Expression; +import com.oceanbase.tools.sqlparser.statement.createtable.CreateTable; +import com.oceanbase.tools.sqlparser.statement.createtable.Partition; +import com.oceanbase.tools.sqlparser.statement.createtable.PartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubListPartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionOption; +import com.oceanbase.tools.sqlparser.statement.createtable.SubRangePartitionElement; +import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/11/29 14:34 + * @since: 4.3.3 + */ +@Slf4j +public abstract class BaseOBGetDBTableByParser implements GetDBTableByParser { + + public final DBTablePartition getPartition() { + DBTablePartition partition = new DBTablePartition(); + DBTablePartitionOption partitionOption = new DBTablePartitionOption(); + partitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); + partition.setPartitionOption(partitionOption); + List partitionDefinitions = new ArrayList<>(); + partition.setPartitionDefinitions(partitionDefinitions); + + if (Objects.isNull(getCreateTableStmt())) { + partition.setWarning("Failed to parse table ddl"); + return partition; + } + Partition partitionStmt = getCreateTableStmt().getPartition(); + if (Objects.isNull(partitionStmt)) { + return partition; + } + parsePartitionStmt(partition, partitionStmt); + + /** + * In order to adapt to the front-end only the expression field is used for Hash、List and Range + * partition types + */ + if (Objects.nonNull(partition.getPartitionOption().getType()) + && partition.getPartitionOption().getType().supportExpression() + && StringUtils.isBlank(partition.getPartitionOption().getExpression())) { + List columnNames = partition.getPartitionOption().getColumnNames(); + if (!columnNames.isEmpty()) { + partition.getPartitionOption().setExpression(String.join(", ", columnNames)); + } + } + fillSubPartitions(partition, partitionStmt); + return partition; + } + + private void fillSubPartitions(@NonNull DBTablePartition partition, @NonNull Partition partitionStmt) { + if (partitionStmt.getSubPartitionOption() == null) { + return; + } + DBTablePartition subPartition = new DBTablePartition(); + partition.setSubpartition(subPartition); + DBTablePartitionOption subPartitionOption = new DBTablePartitionOption(); + subPartitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); + subPartition.setPartitionOption(subPartitionOption); + List subPartitionDefinitions = new ArrayList<>(); + subPartition.setPartitionDefinitions(subPartitionDefinitions); + partition.setSubpartitionTemplated(partitionStmt.getSubPartitionOption().getTemplates() != null); + String type = partitionStmt.getSubPartitionOption().getType(); + DBTablePartitionType subDBTablePartitionType = DBTablePartitionType.fromValue(type); + if (DBTablePartitionType.NOT_PARTITIONED == subDBTablePartitionType) { + partition.setWarning("not support this subpartition type, type: " + type); + return; + } + fillSubPartitionOption(partitionStmt, subPartitionOption, subDBTablePartitionType); + fillSubPartitionDefinitions(partition, partitionStmt, subPartitionDefinitions, subDBTablePartitionType); + } + + private void fillSubPartitionOption(@NonNull Partition partitionStmt, + @NonNull DBTablePartitionOption subPartitionOption, + @NonNull DBTablePartitionType subDBTablePartitionType) { + subPartitionOption.setType(subDBTablePartitionType); + SubPartitionOption parsedSubPartitionOption = partitionStmt.getSubPartitionOption(); + fillSubPartitionKey(subPartitionOption, subDBTablePartitionType, parsedSubPartitionOption); + /** + * 
+         * subpartitionsNum indicates the number of subpartitions in each partition.
+         * Therefore, subpartitionsNum should be configured only when the subpartitions are templated.
+         * If subpartition is not templated, the subpartitionsNum is not fixed.
+         * such as the following example of non-template subpartition table
+         *
+         * CREATE TABLE ranges_list (col1 INT,col2 INT)
+         *        PARTITION BY RANGE COLUMNS(col1)
+         *        SUBPARTITION BY LIST(col2)
+         *        (PARTITION p0 VALUES LESS THAN(100)
+         *          (SUBPARTITION sp0 VALUES IN(1,3),
+         *           SUBPARTITION sp1 VALUES IN(4,6),
+         *           SUBPARTITION sp2 VALUES IN(7,9)),
+         *         PARTITION p1 VALUES LESS THAN(200)
+         *          (SUBPARTITION sp3 VALUES IN(1,3))
+         *        );
+         *
+         * In this case, the subpartitionsNum is not fixed.
+         *
+ */ + subPartitionOption + .setPartitionsNum( + parsedSubPartitionOption.getTemplates() != null ? parsedSubPartitionOption.getTemplates().size() + : null); + } + + private void fillSubPartitionDefinitions(@NonNull DBTablePartition partition, @NonNull Partition partitionStmt, + @NonNull List subPartitionDefinitions, + @NonNull DBTablePartitionType subDBTablePartitionType) { + List partitionElements = partitionStmt.getPartitionElements(); + if (partitionElements == null || partitionElements.isEmpty()) { + log.warn("no partitions found in partition statement"); + return; + } + for (int i = 0; i < partitionElements.size(); i++) { + DBTablePartitionDefinition partitionDefinition = partition.getPartitionDefinitions().get(i); + PartitionElement partitionElement = partitionElements.get(i); + if (partitionElement == null) { + continue; + } + List subPartitionElements = partitionElement.getSubPartitionElements(); + if (subPartitionElements != null) { + fillNonTemplatedSubPartitionDefinitions(subPartitionDefinitions, subDBTablePartitionType, + partitionElement, partitionDefinition); + } else { + fillTemplatedSubPartitionDefinitions(partitionStmt, subPartitionDefinitions, subDBTablePartitionType, + partitionElement, partitionDefinition); + } + } + } + + private void fillNonTemplatedSubPartitionDefinitions( + @NonNull List subPartitionDefinitions, + @NonNull DBTablePartitionType subDBTablePartitionType, @NonNull PartitionElement partitionElement, + @NonNull DBTablePartitionDefinition partitionDefinition) { + List subPartitionElements = partitionElement.getSubPartitionElements(); + if (subPartitionElements == null || subPartitionElements.isEmpty()) { + log.warn("no non-templated sub-partitions found for partition {}", partitionDefinition.getName()); + return; + } + for (int j = 0; j < subPartitionElements.size(); j++) { + SubPartitionElement subPartitionElement = subPartitionElements.get(j); + if (subPartitionElement == null) { + continue; + } + DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); + fillSubPartitionValue(subPartitionElement, subPartitionDefinition); + subPartitionDefinition.setParentPartitionDefinition(partitionDefinition); + subPartitionDefinition.setName( + removeIdentifiers(subPartitionElement.getRelation())); + subPartitionDefinition.setOrdinalPosition(j); + subPartitionDefinition.setType(subDBTablePartitionType); + subPartitionDefinitions.add(subPartitionDefinition); + } + } + + private void fillTemplatedSubPartitionDefinitions(@NonNull Partition partitionStmt, + @NonNull List subPartitionDefinitions, + @NonNull DBTablePartitionType subDBTablePartitionType, @NonNull PartitionElement partitionElement, + @NonNull DBTablePartitionDefinition partitionDefinition) { + String parentPartitionName = removeIdentifiers(partitionElement.getRelation()); + List templates = partitionStmt.getSubPartitionOption().getTemplates(); + if (templates == null || templates.isEmpty()) { + log.warn("no templated sub-partitions found for partition {}", parentPartitionName); + return; + } + for (int j = 0; j < templates.size(); j++) { + SubPartitionElement subPartitionElement = templates.get(j); + if (subPartitionElement == null) { + continue; + } + DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); + fillSubPartitionValue(subPartitionElement, subPartitionDefinition); + subPartitionDefinition.setParentPartitionDefinition(partitionDefinition); + subPartitionDefinition.setName( + generateTemplateSubPartitionName(parentPartitionName, subPartitionElement.getRelation())); + subPartitionDefinition.setOrdinalPosition(j); + subPartitionDefinition.setType(subDBTablePartitionType); + subPartitionDefinitions.add(subPartitionDefinition); + } + } + + private void fillSubPartitionValue(@NonNull SubPartitionElement subPartitionElement, + @NonNull DBTablePartitionDefinition subPartitionDefinition) { + if (subPartitionElement instanceof SubListPartitionElement) { + SubListPartitionElement subListPartitionElement = + (SubListPartitionElement) subPartitionElement; + List> valuesList = new ArrayList<>(); + for (Expression listExpr : subListPartitionElement.getListExprs()) { + if (listExpr instanceof CollectionExpression) { + valuesList.add( + ((CollectionExpression) listExpr).getExpressionList().stream() + .map(Expression::getText) + .collect(Collectors.toList())); + } else if (listExpr instanceof ConstExpression) { + valuesList.add(Collections.singletonList(listExpr.getText())); + } + } + subPartitionDefinition.setValuesList(valuesList); + } else if (subPartitionElement instanceof SubRangePartitionElement) { + SubRangePartitionElement subRangePartitionElement = + (SubRangePartitionElement) subPartitionElement; + subPartitionDefinition.setMaxValues( + subRangePartitionElement.getRangeExprs().stream().map(Expression::getText) + .collect(Collectors.toList())); + } + } + + protected abstract String generateTemplateSubPartitionName(String partitionName, String subPartitionName); + + protected abstract void fillSubPartitionKey(DBTablePartitionOption subPartitionOption, + DBTablePartitionType subDBTablePartitionType, + SubPartitionOption parsedSubPartitionOption); + + protected abstract void parsePartitionStmt(DBTablePartition partition, Partition partitionStmt); + + protected abstract String removeIdentifiers(String str); + + protected abstract CreateTable getCreateTableStmt(); + + @Override + public List listIndexes() { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listColumns() { + throw new UnsupportedOperationException("Not supported yet"); + } + +} diff --git a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java index 687b0330ef..19625a8dbd 100644 --- a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java +++ b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java @@ -18,16 +18,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.tools.dbbrowser.model.DBConstraintType; import com.oceanbase.tools.dbbrowser.model.DBForeignKeyModifyRule; -import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.dbbrowser.model.DBTableConstraint; -import com.oceanbase.tools.dbbrowser.model.DBTableIndex; import com.oceanbase.tools.dbbrowser.model.DBTablePartition; import com.oceanbase.tools.dbbrowser.model.DBTablePartitionDefinition; import com.oceanbase.tools.dbbrowser.model.DBTablePartitionOption; @@ -47,7 +44,6 @@ import com.oceanbase.tools.sqlparser.statement.createtable.OutOfLineConstraint; import com.oceanbase.tools.sqlparser.statement.createtable.OutOfLineForeignConstraint; import com.oceanbase.tools.sqlparser.statement.createtable.Partition; -import com.oceanbase.tools.sqlparser.statement.createtable.PartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartition; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.SubListPartitionElement; @@ -68,7 +64,7 @@ * @since 4.2.0 */ @Slf4j -public class OBMySQLGetDBTableByParser implements GetDBTableByParser { +public class OBMySQLGetDBTableByParser extends BaseOBGetDBTableByParser { private final CreateTable createTableStmt; private static final char MYSQL_IDENTIFIER_WRAP_CHAR = '`'; @@ -91,8 +87,8 @@ private CreateTable parseTableDDL(String ddl) { } @Override - public List listColumns() { - throw new UnsupportedOperationException("Not supported yet"); + public CreateTable getCreateTableStmt() { + return this.createTableStmt; } /** @@ -188,78 +184,18 @@ public List listConstraints() { return constraints; } - private String removeIdentifiers(String str) { + protected String removeIdentifiers(String str) { return StringUtils.unwrap(str, MYSQL_IDENTIFIER_WRAP_CHAR); } @Override - public List listIndexes() { - throw new UnsupportedOperationException("Not supported yet"); + protected String generateTemplateSubPartitionName(String partitionName, String subPartitionName) { + return removeIdentifiers(partitionName) + 's' + removeIdentifiers(subPartitionName); } @Override - public DBTablePartition getPartition() { - DBTablePartition partition = new DBTablePartition(); - DBTablePartitionOption partitionOption = new DBTablePartitionOption(); - partitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); - partition.setPartitionOption(partitionOption); - List partitionDefinitions = new ArrayList<>(); - partition.setPartitionDefinitions(partitionDefinitions); - - if (Objects.isNull(createTableStmt)) { - partition.setWarning("Failed to parse table ddl"); - return partition; - } - Partition partitionStmt = createTableStmt.getPartition(); - if (Objects.isNull(partitionStmt)) { - return partition; - } - if (partitionStmt instanceof HashPartition) { - parseHashPartitionStmt((HashPartition) partitionStmt, partition); - } else if (partitionStmt instanceof KeyPartition) { - parseKeyPartitionStmt((KeyPartition) partitionStmt, partition); - } else if (partitionStmt instanceof RangePartition) { - parseRangePartitionStmt((RangePartition) partitionStmt, partition); - } else if (partitionStmt instanceof ListPartition) { - parseListPartitionStmt((ListPartition) partitionStmt, partition); - } - - /** - * In order to adapt to the front-end only the expression field is used for Hash、List and Range - * partition types - */ - if (Objects.nonNull(partition.getPartitionOption().getType()) - && partition.getPartitionOption().getType().supportExpression() - && StringUtils.isBlank(partition.getPartitionOption().getExpression())) { - List columnNames = partition.getPartitionOption().getColumnNames(); - if (!columnNames.isEmpty()) { - partition.getPartitionOption().setExpression(String.join(", ", columnNames)); - } - } - fillSubPartitions(partition, partitionStmt); - return partition; - } - - private void fillSubPartitions(DBTablePartition partition, Partition partitionStmt) { - if (partitionStmt.getSubPartitionOption() == null) { - return; - } - DBTablePartition subPartition = new DBTablePartition(); - partition.setSubpartition(subPartition); - DBTablePartitionOption subPartitionOption = new DBTablePartitionOption(); - subPartitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); - subPartition.setPartitionOption(subPartitionOption); - List subPartitionDefinitions = new ArrayList<>(); - subPartition.setPartitionDefinitions(subPartitionDefinitions); - partition.setSubpartitionTemplated(partitionStmt.getSubPartitionOption().getTemplates() != null); - String type = partitionStmt.getSubPartitionOption().getType(); - DBTablePartitionType subDBTablePartitionType = DBTablePartitionType.fromValue(type); - if (DBTablePartitionType.NOT_PARTITIONED == subDBTablePartitionType) { - partition.setWarning("not support this subpartition type, type: " + type); - return; - } - subPartitionOption.setType(subDBTablePartitionType); - SubPartitionOption parsedSubPartitionOption = partitionStmt.getSubPartitionOption(); + protected void fillSubPartitionKey(DBTablePartitionOption subPartitionOption, + DBTablePartitionType subDBTablePartitionType, SubPartitionOption parsedSubPartitionOption) { // When expressions are supported, only single partition keys are supported if (subDBTablePartitionType.supportExpression()) { Expression expression = parsedSubPartitionOption.getSubPartitionTargets().get(0); @@ -275,61 +211,18 @@ private void fillSubPartitions(DBTablePartition partition, Partition partitionSt .map(item -> removeIdentifiers(item.getText())) .collect(Collectors.toList())); } - /** - * 
-         * subpartitionsNum indicates the number of subpartitions in each partition.
-         * Therefore, subpartitionsNum should be configured only when the subpartitions are templated.
-         * If subpartition is not templated, the subpartitionsNum is not fixed.
-         * such as the following example of non-template subpartition table
-         *
-         * CREATE TABLE ranges_list (col1 INT,col2 INT)
-         *        PARTITION BY RANGE COLUMNS(col1)
-         *        SUBPARTITION BY LIST(col2)
-         *        (PARTITION p0 VALUES LESS THAN(100)
-         *          (SUBPARTITION sp0 VALUES IN(1,3),
-         *           SUBPARTITION sp1 VALUES IN(4,6),
-         *           SUBPARTITION sp2 VALUES IN(7,9)),
-         *         PARTITION p1 VALUES LESS THAN(200)
-         *          (SUBPARTITION sp3 VALUES IN(1,3))
-         *        );
-         *
-         * In this case, the subpartitionsNum is not fixed.
-         *
- */ - subPartitionOption - .setPartitionsNum( - parsedSubPartitionOption.getTemplates() != null ? parsedSubPartitionOption.getTemplates().size() - : null); - for (PartitionElement partitionElement : partitionStmt.getPartitionElements()) { - if (partitionElement.getSubPartitionElements() != null) { - // obtain DBTablePartitionDefinitions for non-templated subpartitions - for (int i = 0; i < partitionElement.getSubPartitionElements().size(); i++) { - DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); - SubPartitionElement subPartitionElement = partitionElement.getSubPartitionElements().get(i); - fillSubPartitionValue(subPartitionElement, subPartitionDefinition); - subPartitionDefinition.setName( - removeIdentifiers(subPartitionElement.getRelation())); - subPartitionDefinition.setOrdinalPosition(i); - subPartitionDefinition.setType(subDBTablePartitionType); - subPartitionDefinitions.add(subPartitionDefinition); - } - } else { - // obtain DBTablePartitionDefinitions for templated subpartitions - String parentPartitionName = removeIdentifiers(partitionElement.getRelation()); - List templates = partitionStmt.getSubPartitionOption().getTemplates(); - for (int i = 0; i < templates.size(); i++) { - DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); - SubPartitionElement subPartitionElement = templates.get(i); - fillSubPartitionValue(subPartitionElement, subPartitionDefinition); - // for a templated subpartition table, the naming rule for the subpartition is - // '($part_name)s($subpart_name)'. - subPartitionDefinition.setName( - parentPartitionName + 's' + removeIdentifiers(subPartitionElement.getRelation())); - subPartitionDefinition.setOrdinalPosition(i); - subPartitionDefinition.setType(subDBTablePartitionType); - subPartitionDefinitions.add(subPartitionDefinition); - } - } + } + + @Override + protected void parsePartitionStmt(DBTablePartition partition, Partition partitionStmt) { + if (partitionStmt instanceof HashPartition) { + parseHashPartitionStmt((HashPartition) partitionStmt, partition); + } else if (partitionStmt instanceof KeyPartition) { + parseKeyPartitionStmt((KeyPartition) partitionStmt, partition); + } else if (partitionStmt instanceof RangePartition) { + parseRangePartitionStmt((RangePartition) partitionStmt, partition); + } else if (partitionStmt instanceof ListPartition) { + parseListPartitionStmt((ListPartition) partitionStmt, partition); } } diff --git a/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java b/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java index 3a407f20c7..7701bd0466 100644 --- a/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java +++ b/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java @@ -378,6 +378,8 @@ public void getSubpartition_RangeColumnsAndRange_ColumnKey_Template_Success() { Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -410,6 +412,8 @@ public void getSubpartition_RangeColumnsAndRange_ExpressionKey_Template_Success( Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -448,6 +452,8 @@ public void getSubpartition_RangeAndRange_ColumnKey_NoTemplate_Success() { Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -485,6 +491,8 @@ public void getSubpartition_RangeAndRange_ExpressionKey_NoTemplate_Success() { Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); Assert.assertEquals("UNIX_TIMESTAMP('2021/04/01')", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -521,6 +529,8 @@ public void getSubpartition_RangeColumnsAndRangeColumns_ColumnKey_Template_Succe Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -557,6 +567,8 @@ public void getSubpartition_RangeColumnsAndRangeColumns_ColumnKey_NoTemplate_Suc Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -592,6 +604,8 @@ public void getSubpartition_RangeColumnsAndList_ColumnKey_Template_Success() { Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -625,6 +639,8 @@ public void getSubpartition_RangeColumnsAndList_ExpressionKey_Template_Success() Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -659,6 +675,8 @@ public void getSubpartition_RangeColumnsAndList_ColumnKey_NoTemplate_Success() { Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -691,6 +709,8 @@ public void getSubpartition_RangeColumnsAndList_ExpressionKey_NoTemplate_Success Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -730,6 +750,8 @@ public void getSubpartition_RangeColumnsAndListColumns_ColumnKey_Template_Succes Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(1)); Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(1)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -767,6 +789,8 @@ public void getSubpartition_RangeColumnsAndListColumns_ColumnKey_NoTemplate_Succ Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -801,6 +825,8 @@ public void getSubpartition_RangeColumnsAndHash_ColumnKey_Template_Success() { Assert.assertTrue(partition.getSubpartitionTemplated()); Assert.assertEquals(5L, subpartition.getPartitionOption().getPartitionsNum().longValue()); Assert.assertEquals("p0sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -833,6 +859,8 @@ public void getSubpartition_RangeColumnsAndHash_ExpressionKey_Template_Success() Assert.assertTrue(partition.getSubpartitionTemplated()); Assert.assertEquals(5L, subpartition.getPartitionOption().getPartitionsNum().longValue()); Assert.assertEquals("p0sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -866,6 +894,8 @@ public void getSubpartition_RangeColumnsAndHash_ColumnKey_NoTemplate_Success() { Assert.assertFalse(partition.getSubpartitionTemplated()); Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -897,6 +927,8 @@ public void getSubpartition_RangeColumnsAndHash_ExpressionKey_NoTemplate_Success Assert.assertFalse(partition.getSubpartitionTemplated()); Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -929,6 +961,8 @@ public void getSubpartition_RangeColumnsAndKey_ColumnKey_Template_Success() { Assert.assertTrue(partition.getSubpartitionTemplated()); Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); Assert.assertEquals("p0sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -963,6 +997,8 @@ public void getSubpartition_RangeColumnsAndKey_ColumnKey_NoTemplate_Success() { Assert.assertFalse(partition.getSubpartitionTemplated()); Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } } diff --git a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java index 98bf8a51a9..ddb85be12c 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java +++ b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java @@ -28,13 +28,12 @@ import com.oceanbase.odc.common.util.JdbcOperationsUtil; import com.oceanbase.odc.common.util.StringUtils; -import com.oceanbase.odc.plugin.schema.obmysql.parser.GetDBTableByParser; +import com.oceanbase.odc.plugin.schema.obmysql.parser.BaseOBGetDBTableByParser; import com.oceanbase.tools.dbbrowser.model.DBConstraintDeferability; import com.oceanbase.tools.dbbrowser.model.DBConstraintType; import com.oceanbase.tools.dbbrowser.model.DBForeignKeyModifyRule; import com.oceanbase.tools.dbbrowser.model.DBIndexAlgorithm; import com.oceanbase.tools.dbbrowser.model.DBIndexType; -import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.dbbrowser.model.DBTableConstraint; import com.oceanbase.tools.dbbrowser.model.DBTableIndex; import com.oceanbase.tools.dbbrowser.model.DBTablePartition; @@ -60,13 +59,9 @@ import com.oceanbase.tools.sqlparser.statement.createtable.OutOfLineConstraint; import com.oceanbase.tools.sqlparser.statement.createtable.OutOfLineForeignConstraint; import com.oceanbase.tools.sqlparser.statement.createtable.Partition; -import com.oceanbase.tools.sqlparser.statement.createtable.PartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartition; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartitionElement; -import com.oceanbase.tools.sqlparser.statement.createtable.SubListPartitionElement; -import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionOption; -import com.oceanbase.tools.sqlparser.statement.createtable.SubRangePartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.TableElement; import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; @@ -82,7 +77,7 @@ * @since 4.2.0 */ @Slf4j -public class OBOracleGetDBTableByParser implements GetDBTableByParser { +public class OBOracleGetDBTableByParser extends BaseOBGetDBTableByParser { @Getter private final CreateTable createTableStmt; private final Connection connection; @@ -102,6 +97,11 @@ public OBOracleGetDBTableByParser(@NonNull Connection connection, @NonNull Strin this.createTableStmt = parseTableDDL(schemaName, tableName); } + @Override + public CreateTable getCreateTableStmt() { + return this.createTableStmt; + } + private CreateTable parseTableDDL(@NonNull String schemaName, @NonNull String tableName) { CreateTable statement = null; OracleSqlBuilder builder = new OracleSqlBuilder(); @@ -123,11 +123,6 @@ private CreateTable parseTableDDL(@NonNull String schemaName, @NonNull String ta return statement; } - @Override - public List listColumns() { - throw new UnsupportedOperationException("Not supported yet"); - } - @Override public List listConstraints() { if (this.constraints.size() > 0 || this.createTableStmt == null) { @@ -250,7 +245,7 @@ public List listConstraints() { return constraints; } - private String removeIdentifiers(String str) { + protected String removeIdentifiers(String str) { if (Objects.isNull(str)) { return ""; } @@ -379,152 +374,18 @@ private CreateIndex parseIndexDDL(String ddl) { } @Override - public DBTablePartition getPartition() { - DBTablePartition partition = new DBTablePartition(); - partition.setPartitionOption(new DBTablePartitionOption()); - partition.setPartitionDefinitions(new ArrayList<>()); - - if (Objects.isNull(createTableStmt)) { - partition.setWarning("Failed to parse ob oracle table ddl"); - return partition; - } - Partition partitionStmt = createTableStmt.getPartition(); - if (Objects.isNull(partitionStmt)) { - return partition; - } - if (partitionStmt instanceof HashPartition) { - parseHashPartitionStmt((HashPartition) partitionStmt, partition); - } else if (partitionStmt instanceof RangePartition) { - parseRangePartitionStmt((RangePartition) partitionStmt, partition); - } else if (partitionStmt instanceof ListPartition) { - parseListPartitionStmt((ListPartition) partitionStmt, partition); - } - - /** - * In order to adapt to the front-end only the expression field is used for Hash、List and Range - * partition types - */ - if (Objects.nonNull(partition.getPartitionOption().getType()) - && partition.getPartitionOption().getType().supportExpression() - && StringUtils.isBlank(partition.getPartitionOption().getExpression())) { - List columnNames = partition.getPartitionOption().getColumnNames(); - if (!columnNames.isEmpty()) { - partition.getPartitionOption().setExpression(String.join(", ", columnNames)); - } - } - fillSubPartitions(partition, partitionStmt); - return partition; + protected String generateTemplateSubPartitionName(String partitionName, String subPartitionName) { + return removeIdentifiers(partitionName) + 'S' + removeIdentifiers(subPartitionName); } - private void fillSubPartitions(DBTablePartition partition, Partition partitionStmt) { - if (partitionStmt.getSubPartitionOption() == null) { - return; - } - DBTablePartition subPartition = new DBTablePartition(); - partition.setSubpartition(subPartition); - DBTablePartitionOption subPartitionOption = new DBTablePartitionOption(); - subPartitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); - subPartition.setPartitionOption(subPartitionOption); - List subPartitionDefinitions = new ArrayList<>(); - subPartition.setPartitionDefinitions(subPartitionDefinitions); - partition.setSubpartitionTemplated(partitionStmt.getSubPartitionOption().getTemplates() != null); - String type = partitionStmt.getSubPartitionOption().getType(); - DBTablePartitionType subDBTablePartitionType = DBTablePartitionType.fromValue(type); - if (DBTablePartitionType.NOT_PARTITIONED == subDBTablePartitionType) { - partition.setWarning("not support this subpartition type, type: " + type); - return; - } - subPartitionOption.setType(subDBTablePartitionType); - SubPartitionOption parsedSubPartitionOption = partitionStmt.getSubPartitionOption(); + @Override + protected void fillSubPartitionKey(DBTablePartitionOption subPartitionOption, + DBTablePartitionType subDBTablePartitionType, SubPartitionOption parsedSubPartitionOption) { // expressions are not supported, multiple columns are supported as partition keys subPartitionOption.setColumnNames(parsedSubPartitionOption.getSubPartitionTargets() == null ? null : parsedSubPartitionOption.getSubPartitionTargets().stream() .map(item -> removeIdentifiers(item.getText())) .collect(Collectors.toList())); - - /** - * 
-         * subpartitionsNum indicates the number of subpartitions in each partition.
-         * Therefore, subpartitionsNum should be configured only when the subpartitions are templated.
-         * If subpartition is not templated, the subpartitionsNum is not fixed.
-         * such as the following example of non-template subpartition table
-         *
-         * CREATE TABLE range_range(col1 INT,col2 INT,col3 INT)
-         *        PARTITION BY RANGE(col1)
-         *        SUBPARTITION BY RANGE(col2,col3)
-         *         (PARTITION p0 VALUES LESS THAN(100)
-         *           (SUBPARTITION sp0 VALUES LESS THAN(2020,2020),
-         *            SUBPARTITION sp1 VALUES LESS THAN(2021,2021)
-         *           ),
-         *          PARTITION p1 VALUES LESS THAN(200)
-         *           (SUBPARTITION sp2 VALUES LESS THAN(2020,2020),
-         *            SUBPARTITION sp3 VALUES LESS THAN(2021,2021),
-         *            SUBPARTITION sp4 VALUES LESS THAN(2022,2022)
-         *            )
-         *          );
-         *
-         * In this case, the subpartitionsNum is not fixed.
-         *
- */ - subPartitionOption - .setPartitionsNum( - parsedSubPartitionOption.getTemplates() != null ? parsedSubPartitionOption.getTemplates().size() - : null); - for (PartitionElement partitionElement : partitionStmt.getPartitionElements()) { - if (partitionElement.getSubPartitionElements() != null) { - // obtain DBTablePartitionDefinitions for non-templated subpartitions - for (int i = 0; i < partitionElement.getSubPartitionElements().size(); i++) { - DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); - SubPartitionElement subPartitionElement = partitionElement.getSubPartitionElements().get(i); - fillSubPartitionValue(subPartitionElement, subPartitionDefinition); - subPartitionDefinition.setName( - removeIdentifiers(subPartitionElement.getRelation())); - subPartitionDefinition.setOrdinalPosition(i); - subPartitionDefinition.setType(subDBTablePartitionType); - subPartitionDefinitions.add(subPartitionDefinition); - } - } else { - // obtain DBTablePartitionDefinitions for templated subpartitions - String parentPartitionName = removeIdentifiers(partitionElement.getRelation()); - List templates = partitionStmt.getSubPartitionOption().getTemplates(); - for (int i = 0; i < templates.size(); i++) { - DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); - SubPartitionElement subPartitionElement = templates.get(i); - fillSubPartitionValue(subPartitionElement, subPartitionDefinition); - // for a templated subpartition table, the naming rule for the subpartition is - // '($part_name)S($subpart_name)'. - subPartitionDefinition.setName( - parentPartitionName + 'S' + removeIdentifiers(subPartitionElement.getRelation())); - subPartitionDefinition.setOrdinalPosition(i); - subPartitionDefinition.setType(subDBTablePartitionType); - subPartitionDefinitions.add(subPartitionDefinition); - } - } - } - } - - private void fillSubPartitionValue(SubPartitionElement subPartitionElement, - DBTablePartitionDefinition subPartitionDefinition) { - if (subPartitionElement instanceof SubRangePartitionElement) { - SubRangePartitionElement subRangePartitionElement = (SubRangePartitionElement) subPartitionElement; - subPartitionDefinition.setMaxValues( - subRangePartitionElement.getRangeExprs().stream().map(Expression::getText) - .collect(Collectors.toList())); - } else if (subPartitionElement instanceof SubListPartitionElement) { - SubListPartitionElement subListPartitionElement = (SubListPartitionElement) subPartitionElement; - List> valuesList = new ArrayList<>(); - for (Expression listExpr : subListPartitionElement.getListExprs()) { - if (listExpr instanceof CollectionExpression) { - valuesList.add( - ((CollectionExpression) listExpr).getExpressionList().stream() - .map(Expression::getText) - .collect(Collectors.toList())); - } else if (listExpr instanceof ConstExpression) { - valuesList.add(Collections.singletonList(listExpr.getText())); - } - } - subPartitionDefinition.setValuesList(valuesList); - } } private void parseHashPartitionStmt(HashPartition statement, DBTablePartition partition) { @@ -606,4 +467,16 @@ private void parseListPartitionStmt(ListPartition statement, DBTablePartition pa } } + + @Override + protected void parsePartitionStmt(DBTablePartition partition, Partition partitionStmt) { + if (partitionStmt instanceof HashPartition) { + parseHashPartitionStmt((HashPartition) partitionStmt, partition); + } else if (partitionStmt instanceof RangePartition) { + parseRangePartitionStmt((RangePartition) partitionStmt, partition); + } else if (partitionStmt instanceof ListPartition) { + parseListPartitionStmt((ListPartition) partitionStmt, partition); + } + } + } diff --git a/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java b/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java index dbfe201610..3cc07ae575 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java +++ b/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java @@ -188,6 +188,8 @@ public void getSubPartition_TemplateSingleRangeMultipleRange_Success() { Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -212,6 +214,8 @@ public void getSubPartition_TemplateSingleRangeSingleRange_Success() { Assert.assertEquals("P0SMP0", subpartition.getPartitionDefinitions().get(0).getName()); Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -236,6 +240,8 @@ public void getSubPartition_TemplateSingleHashMultipleList_Success() { Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(1)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -258,6 +264,8 @@ public void getSubPartition_TemplateSingleHashSingleList_Success() { Assert.assertEquals("P0SSP0", subpartition.getPartitionDefinitions().get(0).getName()); Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -279,6 +287,8 @@ public void getSubPartition_TemplateSingleHashMultipleHash_Success() { Assert.assertNull(subpartition.getPartitionOption().getExpression()); Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); Assert.assertEquals("P0SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -299,6 +309,8 @@ public void getSubPartition_TemplateSingleHashSingleHash_Success() { Assert.assertNull(subpartition.getPartitionOption().getExpression()); Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); Assert.assertEquals("P0SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -325,6 +337,8 @@ public void getSubPartition_SingleRangeMultipleRange_Success() { Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -349,6 +363,8 @@ public void getSubPartition_SingleRangeSingleRange_Success() { Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -373,6 +389,8 @@ public void getSubPartition_SingleHashMultipleList_Success() { Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(1)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -395,6 +413,8 @@ public void getSubPartition_SingleHashSingleList_Success() { Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -415,6 +435,8 @@ public void getSubPartition_SingleListMultipleHash_Success() { Assert.assertNull(subpartition.getPartitionOption().getExpression()); Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test @@ -435,6 +457,8 @@ public void getSubPartition_SingleListSingleHash_Success() { Assert.assertNull(subpartition.getPartitionOption().getExpression()); Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test From 3da0271a2c2e8605cda79ad8052ebe8d3e96998e Mon Sep 17 00:00:00 2001 From: LioRoger Date: Wed, 4 Dec 2024 11:29:51 +0800 Subject: [PATCH 043/118] fix(task): rollback DestroyExecutorJob to do destroy job action --- .../task/schedule/StdJobScheduler.java | 14 ++- .../schedule/daemon/DestroyExecutorJob.java | 114 ++++++++++++++++++ .../task/service/StdTaskFrameworkService.java | 15 --- 3 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/StdJobScheduler.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/StdJobScheduler.java index e6718384fb..5cae71ebf3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/StdJobScheduler.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/StdJobScheduler.java @@ -49,10 +49,12 @@ import com.oceanbase.odc.service.task.constants.JobConstants; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.enums.TaskMonitorMode; +import com.oceanbase.odc.service.task.enums.TaskRunMode; import com.oceanbase.odc.service.task.exception.JobException; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; import com.oceanbase.odc.service.task.listener.DefaultJobCallerListener; import com.oceanbase.odc.service.task.schedule.daemon.CheckRunningJob; +import com.oceanbase.odc.service.task.schedule.daemon.DestroyExecutorJob; import com.oceanbase.odc.service.task.schedule.daemon.DestroyResourceJob; import com.oceanbase.odc.service.task.schedule.daemon.DoCancelingJob; import com.oceanbase.odc.service.task.schedule.daemon.PullTaskResultJob; @@ -201,6 +203,7 @@ private void initDaemonJob() { initStartPreparingJob(); initDoCancelingJob(); initDestroyExecutorJob(); + initDestroyResource(); } private void initCheckRunningJob() { @@ -239,7 +242,16 @@ private void initDestroyExecutorJob() { String key = "destroyExecutorJob"; initCronJob(key, configuration.getTaskFrameworkProperties().getDestroyExecutorJobCronExpression(), - DestroyResourceJob.class); + DestroyExecutorJob.class); + } + + private void initDestroyResource() { + if (configuration.getTaskFrameworkProperties().getRunMode() == TaskRunMode.K8S) { + String key = "destroyResourceJob"; + initCronJob(key, + configuration.getTaskFrameworkProperties().getDestroyExecutorJobCronExpression(), + DestroyResourceJob.class); + } } private void initCronJob(String key, String cronExpression, Class jobClass) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java new file mode 100644 index 0000000000..0e9ae03fd4 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.schedule.daemon; + +import java.text.MessageFormat; +import java.util.Map; +import java.util.Optional; + +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.springframework.data.domain.Page; + +import com.oceanbase.odc.core.alarm.AlarmEventNames; +import com.oceanbase.odc.core.alarm.AlarmUtils; +import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.service.task.config.JobConfiguration; +import com.oceanbase.odc.service.task.config.JobConfigurationHolder; +import com.oceanbase.odc.service.task.config.JobConfigurationValidator; +import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; +import com.oceanbase.odc.service.task.constants.JobConstants; +import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.exception.TaskRuntimeException; +import com.oceanbase.odc.service.task.schedule.JobIdentity; +import com.oceanbase.odc.service.task.service.TaskFrameworkService; + +import cn.hutool.core.util.StrUtil; +import lombok.extern.slf4j.Slf4j; + +/** + * @author yaobin + * @date 2024-01-22 + * @since 4.2.4 + */ +@Slf4j +@DisallowConcurrentExecution +public class DestroyExecutorJob implements Job { + + private JobConfiguration configuration; + + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + configuration = JobConfigurationHolder.getJobConfiguration(); + JobConfigurationValidator.validComponent(); + + // scan terminate job + TaskFrameworkService taskFrameworkService = configuration.getTaskFrameworkService(); + TaskFrameworkProperties taskFrameworkProperties = configuration.getTaskFrameworkProperties(); + Page jobs = taskFrameworkService.findTerminalJob(0, + taskFrameworkProperties.getSingleFetchDestroyExecutorJobRows()); + jobs.forEach(a -> { + try { + destroyExecutor(taskFrameworkService, a); + } catch (Throwable e) { + log.warn("Try to destroy failed, jobId={}.", a.getId(), e); + } + }); + } + + private void destroyExecutor(TaskFrameworkService taskFrameworkService, JobEntity jobEntity) { + getConfiguration().getTransactionManager().doInTransactionWithoutResult(() -> { + JobEntity lockedEntity = taskFrameworkService.findWithPessimisticLock(jobEntity.getId()); + if (lockedEntity.getStatus().isTerminated() && lockedEntity.getExecutorIdentifier() != null) { + log.info("Job prepare destroy executor, jobId={},status={}.", lockedEntity.getId(), + lockedEntity.getStatus()); + try { + getConfiguration().getJobDispatcher().finish(JobIdentity.of(lockedEntity.getId())); + } catch (JobException e) { + log.warn("Destroy executor occur error, jobId={}: ", lockedEntity.getId(), e); + if (e.getMessage() != null && + !e.getMessage().startsWith(JobConstants.ODC_EXECUTOR_CANNOT_BE_DESTROYED)) { + Map eventMessage = AlarmUtils.createAlarmMapBuilder() + .item(AlarmUtils.ORGANIZATION_NAME, + Optional.ofNullable(jobEntity.getOrganizationId()).map( + Object::toString).orElse(StrUtil.EMPTY)) + .item(AlarmUtils.TASK_JOB_ID_NAME, String.valueOf(jobEntity.getId())) + .item(AlarmUtils.MESSAGE_NAME, + MessageFormat.format("Job executor destroy failed, jobId={0}, message={1}", + lockedEntity.getId(), e.getMessage())) + .build(); + AlarmUtils.alarm(AlarmEventNames.TASK_EXECUTOR_DESTROY_FAILED, eventMessage); + } + throw new TaskRuntimeException(e); + } + log.info("Job destroy executor succeed, jobId={}, status={}.", lockedEntity.getId(), + lockedEntity.getStatus()); + } else if (lockedEntity.getStatus().isTerminated() && lockedEntity.getExecutorIdentifier() == null) { + // It is necessary to update the finish time when the job is terminated but the + // executorIdentifier is null, otherwise, the job cannot be released. + log.info("Executor not found, updating executor to destroyed,jobId={}", lockedEntity.getId()); + taskFrameworkService.updateExecutorToDestroyed(lockedEntity.getId()); + } + }); + } + + + private JobConfiguration getConfiguration() { + return configuration; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java index fc5b14c688..276647e5a2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java @@ -65,12 +65,8 @@ import com.oceanbase.odc.metadb.task.JobAttributeRepository; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.metadb.task.JobRepository; -import com.oceanbase.odc.service.resource.ResourceID; import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.resource.ResourceState; -import com.oceanbase.odc.service.task.caller.ExecutorIdentifier; -import com.oceanbase.odc.service.task.caller.ExecutorIdentifierParser; -import com.oceanbase.odc.service.task.caller.ResourceIDUtil; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; import com.oceanbase.odc.service.task.constants.JobAttributeEntityColumn; import com.oceanbase.odc.service.task.constants.JobEntityColumn; @@ -389,8 +385,6 @@ private void doRefreshResult(Long id) throws JobException { protected void handleTaskResultInner(JobEntity jobEntity, TaskResult result) { JobStatus expectedJobStatus = jobStatusFsm.determinateJobStatus(jobEntity.getStatus(), result.getStatus()); int rows = updateTaskResult(result, jobEntity, expectedJobStatus); - // release resource - tryReleaseResource(jobEntity, expectedJobStatus.isTerminated()); if (rows == 0) { log.warn("Update task result failed, the job may finished or deleted already, jobId={}", jobEntity.getId()); return; @@ -462,15 +456,6 @@ public boolean refreshLogMetaForCancelJob(Long id) { } } - protected void tryReleaseResource(JobEntity jobEntity, boolean isJobDone) { - // release resource - if (isJobDone && TaskRunMode.K8S == jobEntity.getRunMode()) { - ExecutorIdentifier executorIdentifier = ExecutorIdentifierParser.parser(jobEntity.getExecutorIdentifier()); - ResourceID resourceID = ResourceIDUtil.getResourceID(executorIdentifier, jobEntity); - resourceManager.release(resourceID); - } - } - private boolean updateHeartbeatTime(Long id) { int rows = jobRepository.updateHeartbeatTime(id, JobStatus.RUNNING); if (rows > 0) { From c6da51f3d8b732aa71a3c12b31e0e4aafbd628fe Mon Sep 17 00:00:00 2001 From: LioRoger Date: Wed, 4 Dec 2024 16:45:54 +0800 Subject: [PATCH 044/118] fix(task): add index to resource_resource table --- .../common/V_4_3_3_1__alter_resource_resource_add_index.sql | 4 ++++ .../com/oceanbase/odc/metadb/resource/ResourceEntity.java | 2 ++ .../odc/service/task/service/StdTaskFrameworkService.java | 5 ++++- 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_resource_resource_add_index.sql diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_resource_resource_add_index.sql b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_resource_resource_add_index.sql new file mode 100644 index 0000000000..ddf84db49a --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_resource_resource_add_index.sql @@ -0,0 +1,4 @@ +-- +-- Add constraint (resource_type, status) to `resource_resource` table +-- +alter table `resource_resource` add index `type_status`(`resource_type`, `status`); \ No newline at end of file diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java index 0c89b3f515..fd5acb3bed 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java @@ -48,6 +48,8 @@ public class ResourceEntity { public static final String CREATE_TIME = "createTime"; public static final String STATUS = "status"; + public static final String TYPE = "resourceType"; + /** * id for task */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java index 276647e5a2..07a0b8819b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java @@ -78,6 +78,7 @@ import com.oceanbase.odc.service.task.listener.DefaultJobProcessUpdateEvent; import com.oceanbase.odc.service.task.listener.JobTerminateEvent; import com.oceanbase.odc.service.task.processor.result.ResultProcessor; +import com.oceanbase.odc.service.task.resource.DefaultResourceOperatorBuilder; import com.oceanbase.odc.service.task.schedule.JobDefinition; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.state.JobStatusFsm; @@ -183,7 +184,9 @@ public Page findAbandonedResource(int page, int size) { Specification specification = SpecificationUtil.columnLate(ResourceEntity.CREATE_TIME, JobDateUtils.getCurrentDateSubtractDays(RECENT_DAY)); Specification condition = Specification.where(specification) - .and(SpecificationUtil.columnIn(ResourceEntity.STATUS, Lists.newArrayList(ResourceState.ABANDONED))); + .and(SpecificationUtil.columnEqual(ResourceEntity.STATUS, ResourceState.ABANDONED)) + .and(SpecificationUtil.columnEqual(ResourceEntity.TYPE, + DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE)); return resourceRepository.findAll(condition, PageRequest.of(page, size)); } From bd544dea38133a90c365e9a5c012925f33d4060f Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Thu, 5 Dec 2024 17:07:30 +0800 Subject: [PATCH 045/118] feat(sso): support saml integration (#3783) * add saml integration * add saml integration * fill secret when testLogin * fill secret when testLogin * fill secret when testLogin * amend test * amend test * fix * fix * fix --- pom.xml | 35 +++- .../common/V_4_3_3_1__alter_integration.sql | 17 ++ server/odc-server/pom.xml | 4 + .../web/controller/v2/SSOController.java | 20 +- server/odc-service/pom.xml | 18 +- .../odc/service/common/util/UrlUtils.java | 17 ++ ...=> IntegrationConfigurationProcessor.java} | 43 +++- ...rationConfigurationProcessorDelegate.java} | 26 ++- .../integration/IntegrationService.java | 26 ++- .../service/integration/SSOCredential.java | 27 +++ .../service/integration/model/Encryption.java | 3 +- .../integration/model/Oauth2Parameter.java | 5 +- .../model/SSOIntegrationConfig.java | 93 +++++---- .../integration/oauth2/SSOEventHandler.java | 10 +- ...StateManager.java => SSOStateManager.java} | 7 +- .../integration/oauth2/TestLoginManager.java | 116 +++++++++-- ...bleRelyingPartyRegistrationRepository.java | 67 +++++++ .../saml/SamlCredentialManager.java | 55 ++++++ .../integration/saml/SamlParameter.java | 170 ++++++++++++++++ .../saml/SamlRegistrationConfigHelper.java | 187 ++++++++++++++++++ .../integration/util/EncryptionUtil.java | 93 +++++++++ .../odc/config/WebSecurityConfiguration.java | 36 ++++ .../service/iam/auth/MappingRuleConvert.java | 47 ++++- ...tomOAuth2AuthorizationRequestResolver.java | 12 +- .../oauth2/OAuth2SecurityConfigureHelper.java | 8 +- ... OAuth2TestLoginAuthenticationFilter.java} | 2 +- .../auth/oauth2/OAuth2UserServiceImpl.java | 7 +- .../iam/auth/oauth2/OidcUserServiceImpl.java | 7 +- .../iam/auth/saml/CustomSamlProvider.java | 60 ++++++ .../iam/auth/saml/DefaultSamlUserService.java | 42 ++++ .../saml/SamlSecurityConfigureHelper.java | 60 ++++++ .../SamlTestLoginAuthenticationFilter.java | 28 +++ .../odc/service/info/WebInfoAdapter.java | 10 +- 33 files changed, 1241 insertions(+), 117 deletions(-) create mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_integration.sql rename server/odc-service/src/main/java/com/oceanbase/odc/service/integration/{IntegrationConfigurationValidator.java => IntegrationConfigurationProcessor.java} (53%) rename server/odc-service/src/main/java/com/oceanbase/odc/service/integration/{IntegrationConfigurationValidatorDelegate.java => IntegrationConfigurationProcessorDelegate.java} (62%) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/integration/SSOCredential.java rename server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/{Oauth2StateManager.java => SSOStateManager.java} (95%) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/AddableRelyingPartyRegistrationRepository.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlCredentialManager.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlParameter.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlRegistrationConfigHelper.java rename server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/{OAuth2AbstractTestLoginAuthenticationFilter.java => OAuth2TestLoginAuthenticationFilter.java} (90%) create mode 100644 server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/CustomSamlProvider.java create mode 100644 server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/DefaultSamlUserService.java create mode 100644 server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlSecurityConfigureHelper.java create mode 100644 server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlTestLoginAuthenticationFilter.java diff --git a/pom.xml b/pom.xml index e6ada027ba..ed7c8770f6 100644 --- a/pom.xml +++ b/pom.xml @@ -150,6 +150,9 @@ 3.9.0 4.11.0 5.13.3.202401111512-r + 1.76 + 1.70 + 1.15 1.9.5 2.7.12 @@ -193,6 +196,11 @@ groovy-all ${groovy-all.version} + + commons-codec + commons-codec + ${commons-codec.version} + io.micrometer micrometer-core @@ -459,6 +467,12 @@ spring-security-oauth2-client ${spring-security.version} + + + org.springframework.security + spring-security-saml2-service-provider + ${spring-security.version} + org.springframework.security spring-security-crypto @@ -482,7 +496,7 @@ org.bouncycastle bcpkix-jdk15on - ${bcpkix-jdk15on.version} + ${bouncycastle.jdk15.version} org.springframework.integration @@ -499,6 +513,21 @@ spring-cloud-starter-bootstrap ${spring-cloud-starter-bootstrap.version} + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.jdk18.version} + + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.jdk18.version} + + + org.bouncycastle + bcutil-jdk18on + ${bouncycastle.jdk18.version} + org.springframework.cloud spring-cloud-config-server @@ -984,9 +1013,7 @@ org.bouncycastle bcprov-jdk15on - - - 1.70 + ${bouncycastle.jdk15.version} org.hamcrest diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_integration.sql b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_integration.sql new file mode 100644 index 0000000000..3f84b6a06a --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_integration.sql @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +alter table `integration_integration` modify column `secret` mediumtext DEFAULT null; \ No newline at end of file diff --git a/server/odc-server/pom.xml b/server/odc-server/pom.xml index 6566807cc8..d940a825b2 100644 --- a/server/odc-server/pom.xml +++ b/server/odc-server/pom.xml @@ -35,6 +35,10 @@ + + commons-codec + commons-codec + com.h2database h2 diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/SSOController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/SSOController.java index 14ba29ea17..611bf555c1 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/SSOController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/SSOController.java @@ -27,8 +27,10 @@ import com.oceanbase.odc.service.common.response.Responses; import com.oceanbase.odc.service.common.response.SuccessResponse; +import com.oceanbase.odc.service.integration.IntegrationService; +import com.oceanbase.odc.service.integration.SSOCredential; import com.oceanbase.odc.service.integration.model.IntegrationConfig; -import com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import com.oceanbase.odc.service.integration.oauth2.SSOTestInfo; import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; import com.oceanbase.odc.service.state.model.StateName; @@ -42,12 +44,15 @@ public class SSOController { private TestLoginManager testLoginManager; @Autowired - private Oauth2StateManager oauth2StateManager; + private SSOStateManager SSOStateManager; + + @Autowired + private IntegrationService integrationService; @PostMapping(value = "/test/start") public SuccessResponse addTestClientRegistration(@RequestBody IntegrationConfig config, - @RequestParam(required = false) String type) { - return Responses.ok(testLoginManager.getSSOTestInfo(config, type)); + @RequestParam(required = false) String type, @RequestParam(required = false) String odcBackUrl) { + return Responses.ok(testLoginManager.getSSOTestInfo(config, type, odcBackUrl)); } /** @@ -65,7 +70,12 @@ public SuccessResponse testUserInfo(String testId) { @GetMapping("/state") @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#state") public SuccessResponse> getTestClientInfo(@RequestParam String state) { - return Responses.ok(oauth2StateManager.getStateParameters(state)); + return Responses.ok(SSOStateManager.getStateParameters(state)); + } + + @GetMapping("/credential") + public SuccessResponse generateSSOCredential() { + return Responses.ok(integrationService.generateSSOCredential()); } } diff --git a/server/odc-service/pom.xml b/server/odc-service/pom.xml index 0b4a9d09ec..1590a546b1 100644 --- a/server/odc-service/pom.xml +++ b/server/odc-service/pom.xml @@ -1,6 +1,6 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 com.oceanbase @@ -116,6 +116,18 @@ org.bouncycastle bcpkix-jdk15on + + org.bouncycastle + bcprov-jdk18on + + + org.bouncycastle + bcpkix-jdk18on + + + org.bouncycastle + bcutil-jdk18on + org.springframework.cloud spring-cloud-context @@ -141,6 +153,10 @@ spring-security-oauth2-jose 5.7.0 + + org.springframework.security + spring-security-saml2-service-provider + org.springframework.security spring-security-web diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java index c158a7de5d..3178971558 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.common.util; +import java.net.URL; import java.net.URLDecoder; import java.net.URLEncoder; import java.util.ArrayList; @@ -69,6 +70,22 @@ public static String encode(String str) { } } + @SneakyThrows + public static String getUrlHost(String targetUrl) { + URL url = new URL(targetUrl); + StringBuilder host = new StringBuilder(); + host.append(url.getProtocol()).append("://").append(url.getHost()); + int port = url.getPort(); + if (port <= 0) { + return host.toString(); + } + if ("http".equals(url.getProtocol()) && port != 80 || + "https".equals(url.getProtocol()) && port != 443) { + host.append(":").append(url.getPort()); + } + return host.toString(); + } + @Nullable public static String getQueryParameterFirst(String url, String parameterName) { List strings = getQueryParameter(url, parameterName); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessor.java similarity index 53% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidator.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessor.java index 5c97f784eb..2f9efbdcd1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessor.java @@ -36,6 +36,9 @@ import com.oceanbase.odc.service.integration.model.IntegrationConfig; import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; import com.oceanbase.odc.service.integration.model.SqlInterceptorProperties; +import com.oceanbase.odc.service.integration.saml.SamlCredentialManager; +import com.oceanbase.odc.service.integration.saml.SamlParameter; +import com.oceanbase.odc.service.integration.saml.SamlParameter.SecretInfo; /** * @author gaoda.xy @@ -43,22 +46,58 @@ */ @Component @Validated -public class IntegrationConfigurationValidator { +public class IntegrationConfigurationProcessor { @Autowired private IntegrationRepository integrationRepository; + @Autowired + private SamlCredentialManager samlCredentialManager; + public void check(@NotNull @Valid ApprovalProperties properties) {} public void check(@NotNull @Valid SqlInterceptorProperties properties) {} - public void checkAndFillConfig(@NotNull @Valid IntegrationConfig config, Long organizationId, Boolean enabled, + public void checkAndFillConfig(@NotNull @Valid IntegrationConfig config, @Nullable IntegrationConfig savedConfig, + Long organizationId, Boolean enabled, @Nullable Long integrationId) { SSOIntegrationConfig ssoIntegrationConfig = SSOIntegrationConfig.of(config, organizationId); + fillSamlSecret(config, savedConfig, organizationId, ssoIntegrationConfig); config.setConfiguration(JsonUtils.toJson(ssoIntegrationConfig)); checkNotEnabledInDbBeforeSave(enabled, organizationId, integrationId); } + public void fillSamlSecret(IntegrationConfig config, IntegrationConfig savedConfig, Long organizationId, + SSOIntegrationConfig ssoIntegrationConfig) { + if ("SAML".equals(ssoIntegrationConfig.getType()) && config.getEncryption().getSecret() == null) { + SecretInfo secretInfo = new SecretInfo(); + SamlParameter newParameter = (SamlParameter) ssoIntegrationConfig.getSsoParameter(); + if (savedConfig != null) { + SecretInfo savedSecretInfo = + JsonUtils.fromJson(savedConfig.getEncryption().getSecret(), SecretInfo.class); + SSOIntegrationConfig savedSsoConfig = SSOIntegrationConfig.of(savedConfig, organizationId); + SamlParameter savedParameter = (SamlParameter) savedSsoConfig.getSsoParameter(); + if (Objects.equals(savedParameter.getSigning().getCertificate(), + newParameter.getSigning().getCertificate())) { + secretInfo.setSigningPrivateKey(savedSecretInfo.getSigningPrivateKey()); + } + if (Objects.equals(savedParameter.getDecryption().getCertificate(), + newParameter.getDecryption().getCertificate())) { + secretInfo.setDecryptionPrivateKey(savedSecretInfo.getDecryptionPrivateKey()); + } + } + String certificate = newParameter.getSigning().getCertificate(); + if (secretInfo.getSigningPrivateKey() == null && certificate != null) { + secretInfo.setSigningPrivateKey(samlCredentialManager.getPrivateKeyByCert(certificate)); + } + String decryptionCertificate = newParameter.getDecryption().getCertificate(); + if (secretInfo.getDecryptionPrivateKey() == null && decryptionCertificate != null) { + secretInfo.setDecryptionPrivateKey(samlCredentialManager.getPrivateKeyByCert(decryptionCertificate)); + } + config.getEncryption().setSecret(JsonUtils.toJson(secretInfo)); + } + } + public void checkNotEnabledInDbBeforeSave(Boolean enabled, Long organizationId, @Nullable Long integrationId) { if (Boolean.TRUE.equals(enabled)) { List dbSSO = integrationRepository.findByTypeAndOrganizationId(SSO, diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidatorDelegate.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessorDelegate.java similarity index 62% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidatorDelegate.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessorDelegate.java index dcad7029e3..79545da5a3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidatorDelegate.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessorDelegate.java @@ -15,6 +15,10 @@ */ package com.oceanbase.odc.service.integration; +import java.util.Optional; + +import javax.annotation.Nullable; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -25,22 +29,24 @@ import com.oceanbase.odc.service.integration.model.SqlInterceptorProperties; @Component -public class IntegrationConfigurationValidatorDelegate { +public class IntegrationConfigurationProcessorDelegate { @Autowired - private IntegrationConfigurationValidator configurationValidator; + private IntegrationConfigurationProcessor configurationValidator; @Autowired private AuthenticationFacade authenticationFacade; - public void preProcessConfig(IntegrationConfig config) { - if (config.getType() == IntegrationType.APPROVAL) { - configurationValidator.check(ApprovalProperties.from(config)); - } else if (config.getType() == IntegrationType.SQL_INTERCEPTOR) { - configurationValidator.check(SqlInterceptorProperties.from(config)); - } else if (config.getType() == IntegrationType.SSO) { - configurationValidator.checkAndFillConfig(config, authenticationFacade.currentOrganizationId(), - config.getEnabled(), null); + public void preProcessConfig(IntegrationConfig newConfig, @Nullable IntegrationConfig savedConfig) { + if (newConfig.getType() == IntegrationType.APPROVAL) { + configurationValidator.check(ApprovalProperties.from(newConfig)); + } else if (newConfig.getType() == IntegrationType.SQL_INTERCEPTOR) { + configurationValidator.check(SqlInterceptorProperties.from(newConfig)); + } else if (newConfig.getType() == IntegrationType.SSO) { + configurationValidator.checkAndFillConfig(newConfig, savedConfig, + authenticationFacade.currentOrganizationId(), + newConfig.getEnabled(), + Optional.ofNullable(savedConfig).map(IntegrationConfig::getId).orElse(null)); } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java index 8181172512..64a77e37e9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java @@ -74,6 +74,7 @@ import com.oceanbase.odc.service.integration.model.QueryIntegrationParams; import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; import com.oceanbase.odc.service.integration.model.SqlInterceptorProperties; +import com.oceanbase.odc.service.integration.saml.SamlCredentialManager; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -91,7 +92,7 @@ public class IntegrationService { private AuthenticationFacade authenticationFacade; @Autowired - private IntegrationConfigurationValidatorDelegate integrationConfigurationValidatorDelegate; + private IntegrationConfigurationProcessorDelegate integrationConfigurationProcessorDelegate; @Autowired private HorizontalDataPermissionValidator permissionValidator; @@ -117,6 +118,9 @@ public class IntegrationService { @Autowired private CacheManager defaultCacheManager; + @Autowired + private SamlCredentialManager samlCredentialManager; + @PreAuthenticate(actions = "create", resourceType = "ODC_INTEGRATION", isForAll = true) public Boolean exists(@NotBlank String name, @NotNull IntegrationType type) { Long organizationId = authenticationFacade.currentOrganizationId(); @@ -132,7 +136,7 @@ public IntegrationConfig create(@NotNull @Valid IntegrationConfig config) { .findByNameAndTypeAndOrganizationId(config.getName(), config.getType(), organizationId); PreConditions.validNoDuplicated(ResourceType.ODC_EXTERNAL_APPROVAL, "name", config.getName(), existsEntity::isPresent); - integrationConfigurationValidatorDelegate.preProcessConfig(config); + integrationConfigurationProcessorDelegate.preProcessConfig(config, null); Encryption encryption = config.getEncryption(); encryption.check(); applicationContext.publishEvent(IntegrationEvent.createPreCreate(config)); @@ -214,12 +218,13 @@ public IntegrationConfig delete(@NotNull Long id) { @PreAuthenticate(actions = "update", resourceType = "ODC_INTEGRATION", indexOfIdParam = 0) public IntegrationConfig update(@NotNull Long id, @NotNull @Valid IntegrationConfig config) { IntegrationEntity entity = nullSafeGet(id); - permissionValidator.checkCurrentOrganization(new IntegrationConfig(entity)); + IntegrationConfig saveConfig = getDecodeConfig(entity); + permissionValidator.checkCurrentOrganization(saveConfig); if (Boolean.TRUE.equals(entity.getBuiltin())) { throw new UnsupportedException(ErrorCodes.IllegalOperation, new Object[] {"builtin integration"}, "Operation on builtin integration is not allowed"); } - integrationConfigurationValidatorDelegate.preProcessConfig(config); + integrationConfigurationProcessorDelegate.preProcessConfig(config, saveConfig); Encryption encryption = config.getEncryption(); applicationContext.publishEvent( IntegrationEvent.createPreUpdate(config, new IntegrationConfig(entity), entity.getSalt())); @@ -239,6 +244,13 @@ public IntegrationConfig update(@NotNull Long id, @NotNull @Valid IntegrationCon return new IntegrationConfig(entity); } + public IntegrationConfig getDecodeConfig(IntegrationEntity entity) { + IntegrationConfig integrationConfig = new IntegrationConfig(entity); + String secret = decodeSecret(entity.getSecret(), entity.getSalt(), entity.getOrganizationId()); + integrationConfig.getEncryption().setSecret(secret); + return integrationConfig; + } + @Transactional(rollbackFor = Exception.class) @PreAuthenticate(actions = "update", resourceType = "ODC_INTEGRATION", indexOfIdParam = 0) public IntegrationConfig setEnabled(@NotNull Long id, @NotNull Boolean enabled) { @@ -343,7 +355,6 @@ public SSOIntegrationConfig getSSOIntegrationConfig(IntegrationEntity integratio return ssoIntegrationConfig; } - private String encodeSecret(String plainSecret, String salt, Long organizationId) { if (plainSecret == null) { return null; @@ -352,7 +363,6 @@ private String encodeSecret(String plainSecret, String salt, Long organizationId return encryptor.encrypt(plainSecret); } - @SkipAuthorize("odc internal usage") public String decodeSecret(String encryptedSecret, String salt, Long organizationId) { if (encryptedSecret == null) { @@ -369,4 +379,8 @@ private void updateCache(Long key) { } } + public SSOCredential generateSSOCredential() { + return new SSOCredential(samlCredentialManager.generateCertWithCachedPrivateKey()); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/SSOCredential.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/SSOCredential.java new file mode 100644 index 0000000000..ebfeb40dd5 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/SSOCredential.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.integration; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class SSOCredential { + + String certificate; + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Encryption.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Encryption.java index ab3768627d..a61c02151f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Encryption.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Encryption.java @@ -71,7 +71,8 @@ public EncryptionAlgorithm getAlgorithm() { public enum EncryptionAlgorithm { /** - * No encryption + * No encryption. Especially， for sso integration, raw means that each type sso handle the + * encryption and decryption process themselves, Encryption#secret just store it. */ RAW, diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Oauth2Parameter.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Oauth2Parameter.java index ee50d0a2b5..ff183ce3dc 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Oauth2Parameter.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Oauth2Parameter.java @@ -34,6 +34,7 @@ import com.fasterxml.jackson.annotation.JsonProperty.Access; import com.oceanbase.odc.core.shared.constant.OdcConstants; import com.oceanbase.odc.service.common.util.UrlUtils; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import lombok.Data; import lombok.NonNull; @@ -69,8 +70,8 @@ public class Oauth2Parameter implements SSOParameter { /** * {@link Oauth2Parameter} * - * @see com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager put redirect paramters into - * Oauth2StateManager's cache, default value false to adaptive history data + * @see SSOStateManager put redirect paramters into Oauth2StateManager's cache, default value false + * to adaptive history data */ private Boolean useStateParams = true; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/SSOIntegrationConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/SSOIntegrationConfig.java index 628ba474e1..a2a72afd9b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/SSOIntegrationConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/SSOIntegrationConfig.java @@ -33,6 +33,8 @@ import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.Verify; import com.oceanbase.odc.service.integration.model.Encryption.EncryptionAlgorithm; +import com.oceanbase.odc.service.integration.saml.SamlParameter; +import com.oceanbase.odc.service.integration.saml.SamlParameter.SecretInfo; import lombok.AllArgsConstructor; import lombok.Data; @@ -61,19 +63,12 @@ public class SSOIntegrationConfig implements Serializable { @JsonSubTypes.Type(value = Oauth2Parameter.class, name = "OAUTH2"), @JsonSubTypes.Type(value = OidcParameter.class, names = "OIDC"), @JsonSubTypes.Type(value = LdapParameter.class, names = "LDAP"), + @JsonSubTypes.Type(value = SamlParameter.class, names = "SAML"), }) SSOParameter ssoParameter; MappingRule mappingRule; - public boolean isOauth2OrOidc() { - return ImmutableSet.of("OAUTH2", "OIDC").contains(type); - } - - public boolean isLdap() { - return Objects.equals(type, "LDAP"); - } - public static SSOIntegrationConfig of(IntegrationConfig integrationConfig, Long organizationId) { SSOIntegrationConfig ssoIntegrationConfig = JsonUtils.fromJson(integrationConfig.getConfiguration(), SSOIntegrationConfig.class); @@ -96,6 +91,10 @@ public static SSOIntegrationConfig of(IntegrationConfig integrationConfig, Long LdapParameter ldapParameter = (LdapParameter) ssoIntegrationConfig.getSsoParameter(); ldapParameter.setManagerPassword(integrationConfig.getEncryption().getSecret()); break; + case "SAML": + SamlParameter samlParameter = (SamlParameter) ssoIntegrationConfig.getSsoParameter(); + samlParameter.fillSecret(integrationConfig.getEncryption().getSecret()); + break; default: throw new UnsupportedOperationException("unknown type=" + ssoIntegrationConfig.getType()); } @@ -119,11 +118,25 @@ public static String parseRegistrationName(String registrationId) { return split[1]; } + public boolean isOauth2OrOidc() { + return ImmutableSet.of("OAUTH2", "OIDC").contains(type); + } + + public boolean isLdap() { + return Objects.equals(type, "LDAP"); + } + + public boolean isSaml() { + return Objects.equals(type, "SAML"); + } + public String resolveRegistrationId() { if (isOauth2OrOidc()) { return ((Oauth2Parameter) ssoParameter).getRegistrationId(); } else if (isLdap()) { return ((LdapParameter) ssoParameter).getRegistrationId(); + } else if (isSaml()) { + return ((SamlParameter) ssoParameter).getRegistrationId(); } else { throw new UnsupportedOperationException(); @@ -134,40 +147,15 @@ public Long resolveOrganizationId() { return parseOrganizationId(resolveRegistrationId()); } - @Getter - @Setter - @AllArgsConstructor - @NoArgsConstructor - public static class MappingRule { - public static final String USER_PROFILE_NESTED = "NESTED"; - - public static final String TO_BE_REPLACED = "TO_BE_REPLACED"; - - @NotBlank - private String userAccountNameField; - private Set userNickNameField; - private String userProfileViewType; - private String nestedAttributeField; - private List extraInfo; - } - - @Data - public static class CustomAttribute { - private String attributeName; - private String expression; - - public String toAutomationExpression() { - return "extra#" + attributeName; - } - } - public String resolveLoginRedirectUrl() { switch (type) { case "OAUTH2": case "OIDC": return ((Oauth2Parameter) ssoParameter).getLoginRedirectUrl(); + case "SAML": + return ((SamlParameter) ssoParameter).resolveLoginUrl(); default: - throw new UnsupportedOperationException("unknown type=" + type); + return null; } } @@ -188,6 +176,12 @@ public void fillDecryptSecret(String decryptSecret) { if (isLdap()) { ((LdapParameter) ssoParameter).setManagerPassword(decryptSecret); } + if (isSaml()) { + SecretInfo secretInfo = JsonUtils.fromJson(decryptSecret, SecretInfo.class); + SamlParameter samlParameter = (SamlParameter) ssoParameter; + samlParameter.getSigning().setPrivateKey(secretInfo.getSigningPrivateKey()); + samlParameter.getDecryption().setPrivateKey(secretInfo.getDecryptionPrivateKey()); + } } public ClientRegistration toClientRegistration() { @@ -216,4 +210,31 @@ public ClientRegistration toTestClientRegistration(String testType) { } } + @Getter + @Setter + @AllArgsConstructor + @NoArgsConstructor + public static class MappingRule { + public static final String USER_PROFILE_NESTED = "NESTED"; + + public static final String TO_BE_REPLACED = "TO_BE_REPLACED"; + + @NotBlank + private String userAccountNameField; + private Set userNickNameField; + private String userProfileViewType; + private String nestedAttributeField; + private List extraInfo; + } + + @Data + public static class CustomAttribute { + private String attributeName; + private String expression; + + public String toAutomationExpression() { + return "extra#" + attributeName; + } + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOEventHandler.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOEventHandler.java index 693021c119..87b7ff26dd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOEventHandler.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOEventHandler.java @@ -22,7 +22,7 @@ import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.Verify; -import com.oceanbase.odc.service.integration.IntegrationConfigurationValidator; +import com.oceanbase.odc.service.integration.IntegrationConfigurationProcessor; import com.oceanbase.odc.service.integration.IntegrationEvent; import com.oceanbase.odc.service.integration.IntegrationEventHandler; import com.oceanbase.odc.service.integration.IntegrationService; @@ -41,7 +41,7 @@ public class SSOEventHandler implements IntegrationEventHandler { private LdapConfigRegistrationManager ldapConfigRegistrationManager; @Autowired - private IntegrationConfigurationValidator configurationValidator; + private IntegrationConfigurationProcessor integrationConfigurationProcessor; @Autowired private IntegrationService integrationService; @@ -71,7 +71,7 @@ public void preDelete(IntegrationEvent integrationEvent) { public void preUpdate(IntegrationEvent integrationEvent) { IntegrationConfig preConfig = integrationEvent.getPreConfig(); IntegrationConfig currentConfig = integrationEvent.getCurrentConfig(); - configurationValidator.checkNotEnabledInDbBeforeSave(currentConfig.getEnabled(), + integrationConfigurationProcessor.checkNotEnabledInDbBeforeSave(currentConfig.getEnabled(), currentConfig.getOrganizationId(), currentConfig.getId()); // current config will not have secret when it is updated, secret can't change, so use preConfig String decryptSecret = integrationService.decodeSecret(preConfig.getEncryption().getSecret(), @@ -108,7 +108,9 @@ private SSOIntegrationConfig getDecryptConfiguration(IntegrationConfig config, S Verify.verify(config.getType() == SSO, "wrong integration type"); SSOIntegrationConfig ssoIntegrationConfig = JsonUtils.fromJson(config.getConfiguration(), SSOIntegrationConfig.class); - ssoIntegrationConfig.fillDecryptSecret(decryptConfiguration); + if (!ssoIntegrationConfig.isSaml()) { + ssoIntegrationConfig.fillDecryptSecret(decryptConfiguration); + } return ssoIntegrationConfig; } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/Oauth2StateManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOStateManager.java similarity index 95% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/Oauth2StateManager.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOStateManager.java index 514375bb57..fe7f665cb5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/Oauth2StateManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOStateManager.java @@ -23,7 +23,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpMethod; -import org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames; import org.springframework.stereotype.Component; import com.fasterxml.jackson.core.type.TypeReference; @@ -41,7 +40,7 @@ import lombok.SneakyThrows; @Component -public class Oauth2StateManager { +public class SSOStateManager { private final Cache> STATE_PARAM_CACHE = CacheBuilder.newBuilder().maximumSize(1000) .expireAfterWrite(10, TimeUnit.MINUTES) @@ -86,7 +85,7 @@ public Map getStateParameters(String state) { }; @SneakyThrows - public void addStateToCurrentRequestParam() { + public void addStateToCurrentRequestParam(String stateKey) { HttpServletRequest request = WebRequestUtils.getCurrentRequest(); Verify.notNull(request, "request"); // state cached in mem, in the case of multiple nodes，need to rely on the StatefulRoute capability @@ -94,7 +93,7 @@ public void addStateToCurrentRequestParam() { SuccessResponse> stateResponse = requestDispatcher .forward(requestDispatcher.getHostUrl(stateHostGenerator.getHost(), properties.getRequestPort()), HttpMethod.GET, - "/api/v2/sso/state?state=" + request.getParameter(OAuth2ParameterNames.STATE), + "/api/v2/sso/state?state=" + request.getParameter(stateKey), requestDispatcher.getRequestHeaders(request), null) .getContentByType( new TypeReference>>() {}); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java index c232d96fb0..46ec6609c8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.integration.oauth2; +import static com.oceanbase.odc.core.shared.constant.OdcConstants.ODC_BACK_URL_PARAM; import static com.oceanbase.odc.core.shared.constant.OdcConstants.TEST_LOGIN_ID_PARAM; import static com.oceanbase.odc.service.integration.model.SSOIntegrationConfig.parseRegistrationName; @@ -26,6 +27,8 @@ import javax.validation.constraints.NotBlank; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.Registration.Acs; +import org.springframework.security.saml2.core.Saml2ParameterNames; import org.springframework.security.web.util.matcher.AntPathRequestMatcher; import org.springframework.stereotype.Service; @@ -42,6 +45,7 @@ import com.oceanbase.odc.service.common.util.WebRequestUtils; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.iam.auth.TestLoginTerminateException; +import com.oceanbase.odc.service.integration.IntegrationConfigurationProcessor; import com.oceanbase.odc.service.integration.IntegrationService; import com.oceanbase.odc.service.integration.ldap.LdapConfigRegistrationManager; import com.oceanbase.odc.service.integration.model.IntegrationConfig; @@ -49,6 +53,7 @@ import com.oceanbase.odc.service.integration.model.LdapContextHolder; import com.oceanbase.odc.service.integration.model.LdapContextHolder.LdapContext; import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; +import com.oceanbase.odc.service.integration.saml.AddableRelyingPartyRegistrationRepository; import com.oceanbase.odc.service.state.StatefulUuidStateIdGenerator; import lombok.extern.slf4j.Slf4j; @@ -59,8 +64,14 @@ public class TestLoginManager { public static final String REGISTRATION_ID_URI_VARIABLE_NAME = "registrationId"; - public static final AntPathRequestMatcher authorizationRequestMatcher = new AntPathRequestMatcher( + public static final AntPathRequestMatcher oAuth2AuthorizationRequestMatcher = new AntPathRequestMatcher( "/login/oauth2/code" + "/{" + REGISTRATION_ID_URI_VARIABLE_NAME + "}"); + /** + * @see Acs#getEntityId() + */ + public static final AntPathRequestMatcher samlAuthorizationRequestMatcher = + new AntPathRequestMatcher("/login/saml2/sso/{registrationId}"); + private static final AntPathRequestMatcher LDAP_REQUEST_MATCHER = new AntPathRequestMatcher("/api/v2/iam/ldap/login", "POST"); public final Cache testLoginInfoCache = @@ -80,25 +91,63 @@ public class TestLoginManager { @Autowired private StatefulUuidStateIdGenerator statefulUuidStateIdGenerator; + @Autowired + private SSOStateManager ssoStateManager; + + @Autowired(required = false) + private AddableRelyingPartyRegistrationRepository addableRelyingPartyRegistrationRepository; + + @Autowired + private IntegrationConfigurationProcessor integrationConfigurationProcessor; + @SkipAuthorize public static boolean isOAuthTestLoginRequest(HttpServletRequest request) { - String registrationId = resolveRegistrationId(request); + String registrationId = resolveOauthRegistrationId(request); if (registrationId == null) { return false; } return "test".equals(parseRegistrationName(registrationId)); } - private static String resolveRegistrationId(HttpServletRequest request) { - if (authorizationRequestMatcher.matches(request)) { - return authorizationRequestMatcher.matcher(request).getVariables() + private static String resolveOauthRegistrationId(HttpServletRequest request) { + if (oAuth2AuthorizationRequestMatcher.matches(request)) { + return oAuth2AuthorizationRequestMatcher.matcher(request).getVariables() .get(REGISTRATION_ID_URI_VARIABLE_NAME); } return null; } @SkipAuthorize - public void saveOauth2TestIdIfNeed(@NotBlank String loginInfo) { + public static boolean isSamlTestLoginRequest(HttpServletRequest request) { + String registrationId = resolveSamlRegistrationId(request); + if (registrationId == null) { + return false; + } + return "test".equals(parseRegistrationName(registrationId)); + } + + private static String resolveSamlRegistrationId(HttpServletRequest request) { + if (samlAuthorizationRequestMatcher.matches(request)) { + return samlAuthorizationRequestMatcher.matcher(request).getVariables() + .get(REGISTRATION_ID_URI_VARIABLE_NAME); + } + return null; + } + + public void saveSamlInfoIfNeed(String info) { + HttpServletRequest currentRequest = WebRequestUtils.getCurrentRequest(); + Verify.notNull(currentRequest, "currentRequest"); + if (!isSamlTestLoginRequest(currentRequest)) { + return; + } + String testId = WebRequestUtils.getStringValueFromParameterOrAttribute(currentRequest, + Saml2ParameterNames.RELAY_STATE); + Verify.notNull(testId, "testId"); + testLoginInfoCache.put(testId, info); + } + + @SkipAuthorize + public void saveOauth2InfoIfNeed(@NotBlank String loginInfo) { HttpServletRequest currentRequest = WebRequestUtils.getCurrentRequest(); Verify.notNull(currentRequest, "currentRequest"); if (!isOAuthTestLoginRequest(currentRequest)) { @@ -119,17 +168,10 @@ public void saveLdapTestIdIfNeed(@NotBlank String loginInfo) { } @PreAuthenticate(actions = "create", resourceType = "ODC_INTEGRATION", isForAll = true) - public SSOTestInfo getSSOTestInfo(IntegrationConfig config, String type) { + public SSOTestInfo getSSOTestInfo(IntegrationConfig config, String type, String odcBackUrl) { SSOIntegrationConfig ssoConfig = SSOIntegrationConfig.of(config, authenticationFacade.currentOrganizationId()); - Verify.verify(ssoConfig.isOauth2OrOidc() || ssoConfig.isLdap(), "not support sso type"); - if (config.getEncryption().getSecret() == null) { - Optional integration = integrationService.findByTypeAndOrganizationIdAndName( - IntegrationType.SSO, authenticationFacade.currentOrganizationId(), config.getName()); - Verify.verify(integration.isPresent(), "lack of secret"); - IntegrationEntity integrationEntity = integration.get(); - ssoConfig.fillDecryptSecret(integrationService.decodeSecret(integrationEntity.getSecret(), - integrationEntity.getSalt(), integrationEntity.getOrganizationId())); - } + Verify.verify(ssoConfig.isOauth2OrOidc() || ssoConfig.isLdap() || ssoConfig.isSaml(), "not support sso type"); + fillTestSecret(config, ssoConfig); String testId = statefulUuidStateIdGenerator.generateStateId("SSO_TEST_ID"); String redirectUrl = null; String testRegistrationId = null; @@ -146,10 +188,40 @@ public SSOTestInfo getSSOTestInfo(IntegrationConfig config, String type) { } ldapConfigRegistrationManager.addTestConfig(ssoConfig); testRegistrationId = ssoConfig.resolveRegistrationId(); + } else if (ssoConfig.isSaml()) { + if (addableRelyingPartyRegistrationRepository == null) { + throw new UnsupportedOperationException("add test sso is not support"); + } + addableRelyingPartyRegistrationRepository.addTestConfig(ssoConfig); + redirectUrl = UrlUtils.appendQueryParameter(ssoConfig.resolveLoginRedirectUrl(), + TEST_LOGIN_ID_PARAM, testId); + redirectUrl = UrlUtils.appendQueryParameter(redirectUrl, + Saml2ParameterNames.RELAY_STATE, testId); + ssoStateManager.setStateParameter(testId, ODC_BACK_URL_PARAM, odcBackUrl); } return new SSOTestInfo(redirectUrl, testId, testRegistrationId); } + private void fillTestSecret(IntegrationConfig config, SSOIntegrationConfig ssoIntegrationConfig) { + Optional integration = integrationService.findByTypeAndOrganizationIdAndName( + IntegrationType.SSO, authenticationFacade.currentOrganizationId(), config.getName()); + if (ssoIntegrationConfig.isSaml()) { + IntegrationConfig savedConfig = null; + if (integration.isPresent()) { + savedConfig = integrationService.getDecodeConfig(integration.get()); + } + integrationConfigurationProcessor.fillSamlSecret(config, savedConfig, + authenticationFacade.currentOrganizationId(), + ssoIntegrationConfig); + ssoIntegrationConfig.fillDecryptSecret(config.getEncryption().getSecret()); + } else if (config.getEncryption().getSecret() == null) { + Verify.verify(integration.isPresent(), "lack of secret"); + IntegrationEntity integrationEntity = integration.get(); + ssoIntegrationConfig.fillDecryptSecret(integrationService.decodeSecret(integrationEntity.getSecret(), + integrationEntity.getSalt(), integrationEntity.getOrganizationId())); + } + } + @Nullable @PreAuthenticate(actions = "create", resourceType = "ODC_INTEGRATION", isForAll = true) public String getTestUserInfo(String testId) { @@ -186,6 +258,17 @@ public void abortIfLdapTestLogin() { } } + public void abortIfSamlTestLogin() { + HttpServletRequest currentRequest = WebRequestUtils.getCurrentRequest(); + if (currentRequest == null) { + return; + } + if (isSamlTestLoginRequest(currentRequest)) { + ssoStateManager.addStateToCurrentRequestParam(Saml2ParameterNames.RELAY_STATE); + throw new TestLoginTerminateException(); + } + } + @SkipAuthorize public LdapContext loadLdapContext(HttpServletRequest request) { boolean isLdapLogin = LDAP_REQUEST_MATCHER.matches(request); @@ -204,5 +287,4 @@ public LdapContext loadLdapContext(HttpServletRequest request) { LdapContextHolder.setParameter(ldapContext); return ldapContext; } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/AddableRelyingPartyRegistrationRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/AddableRelyingPartyRegistrationRepository.java new file mode 100644 index 0000000000..bce1923cc0 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/AddableRelyingPartyRegistrationRepository.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.integration.saml; + +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistration; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistrationRepository; +import org.springframework.stereotype.Component; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.service.integration.IntegrationService; +import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; + +@Component +public class AddableRelyingPartyRegistrationRepository implements RelyingPartyRegistrationRepository { + + public final Cache testConfigRegistrations = + Caffeine.newBuilder().maximumSize(100).expireAfterWrite(10, TimeUnit.MINUTES).build(); + + @Autowired + IntegrationService integrationService; + + @Override + public RelyingPartyRegistration findByRegistrationId(String registrationId) { + SSOIntegrationConfig configByRegistrationId = findConfigByRegistrationId(registrationId); + return configByRegistrationId == null ? null + : SamlRegistrationConfigHelper.asRegistration(configByRegistrationId); + } + + public SSOIntegrationConfig findConfigByRegistrationId(String registrationId) { + Verify.notBlank(registrationId, "registrationId cannot be empty"); + SSOIntegrationConfig sSoClientRegistration = integrationService.getSSoIntegrationConfig(); + if (sSoClientRegistration == null) { + return testConfigRegistrations.get(registrationId, key -> null); + } + Verify.notNull(sSoClientRegistration, "Saml sSoClientRegistration"); + SamlParameter parameter = (SamlParameter) sSoClientRegistration.getSsoParameter(); + if (Objects.equals(registrationId, parameter.getRegistrationId())) { + return sSoClientRegistration; + } + return testConfigRegistrations.get(registrationId, key -> null); + } + + public void addTestConfig(SSOIntegrationConfig ssoConfig) { + SamlParameter parameter = (SamlParameter) ssoConfig.getSsoParameter(); + parameter.amendTest(); + testConfigRegistrations.put(ssoConfig.resolveRegistrationId(), ssoConfig); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlCredentialManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlCredentialManager.java new file mode 100644 index 0000000000..221019e150 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlCredentialManager.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.integration.saml; + +import static com.oceanbase.odc.service.integration.saml.SamlRegistrationConfigHelper.removeBase64CertificatePem; + +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.concurrent.TimeUnit; + +import org.springframework.stereotype.Component; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.oceanbase.odc.common.lang.Pair; +import com.oceanbase.odc.service.integration.util.EncryptionUtil; + +import lombok.SneakyThrows; + +@Component +public class SamlCredentialManager { + + public final Cache certPrivateKeyCache = + Caffeine.newBuilder().maximumSize(100).expireAfterWrite(5, TimeUnit.MINUTES).build(); + + @SneakyThrows + public String generateCertWithCachedPrivateKey() { + Pair pair = EncryptionUtil.generateKeyPair(); + String privateKeyPem = EncryptionUtil.convertPrivateKeyToPem(pair.left); + String certificate = EncryptionUtil.convertCertificateToPem(pair.right); + certPrivateKeyCache.put(removeBase64CertificatePem(certificate), privateKeyPem); + return certificate; + } + + public String getPrivateKeyByCert(String certificate) { + return certPrivateKeyCache.get(removeBase64CertificatePem(certificate), (key) -> { + throw new RuntimeException("Certificate expired"); + }); + } + +} + diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlParameter.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlParameter.java new file mode 100644 index 0000000000..684cccd14a --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlParameter.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.integration.saml; + +import static com.oceanbase.odc.service.integration.model.SSOIntegrationConfig.parseOrganizationId; + +import javax.annotation.Nullable; + +import org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.AssertingParty; +import org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.Registration.Acs; +import org.springframework.security.saml2.provider.service.registration.Saml2MessageBinding; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty.Access; +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.service.common.util.UrlUtils; +import com.oceanbase.odc.service.integration.model.SSOParameter; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +public class SamlParameter implements SSOParameter { + + private String registrationId; + private String name; + + /** + * @see Acs#getLocation() + */ + private String acsLocation = "{baseUrl}/login/saml2/sso/{registrationId}"; + + /** + * URI to the metadata endpoint for discovery-based configuration. If specified singlesignon + * manually, null is allowed. + * + * @see AssertingParty#getMetadataUri() + */ + @Nullable + private String metadataUri; + + /** + * Ensure request from sp to ldp is not tampered with privateKey generate by server, certificate + * provided by the user only support one Credential + * + * @see org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.Registration.Signing + */ + + private String acsEntityId = "{baseUrl}/saml2/service-provider-metadata/{registrationId}"; + + /** + * @see Acs#getBinding() + */ + private String acsBinding = "POST"; + + + private Signing signing = new Signing(); + + private String providerEntityId; + + /** + * Ensure request from ldp to sp is not tampered with + * + * @see org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.AssertingParty.Verification + */ + private Verification verification = new Verification(); + + private Singlesignon singlesignon = new Singlesignon(); + + /** + * Used for decrypting the SAML authentication request. + * + * @see org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.Decryption + */ + private Decryption decryption = new Decryption(); + + public void fillSecret(String decryptSecret) { + SecretInfo credential = JsonUtils.fromJson(decryptSecret, SecretInfo.class); + if (credential == null) { + return; + } + if (signing != null) { + this.signing.privateKey = credential.getSigningPrivateKey(); + } + if (decryption != null) { + this.decryption.privateKey = credential.getDecryptionPrivateKey(); + } + } + + public String resolveLoginUrl() { + String acsLocationPath = "/login/saml2/sso"; + Verify.verify(acsLocation.contains(acsLocationPath), "invalid acsLocation=" + acsLocation); + String baseUrl = this.acsLocation.split(acsLocationPath)[0]; + return baseUrl + "/saml2/authenticate/" + registrationId; + } + + public void amendTest() { + registrationId = parseOrganizationId(registrationId) + "-" + "test"; + acsLocation = UrlUtils.getUrlHost(acsLocation) + "/login/saml2/sso/" + registrationId; + acsEntityId = UrlUtils.getUrlHost(acsEntityId) + "/saml2/service-provider-metadata/" + registrationId; + } + + /** + * @see AssertingParty#getSinglesignon() + */ + @Data + public static class Singlesignon { + private String url; + + /** + * Whether to redirect or post authentication requests. + * + * @see Saml2MessageBinding + */ + private String binding; + + /** + * Whether to sign authentication requests. + */ + private Boolean signRequest; + } + + @Data + public static class Signing { + /** + * generate by server + */ + @JsonProperty(access = Access.WRITE_ONLY) + private String privateKey; + /** + * provider by user + */ + private String certificate; + } + + @Data + public static class Verification { + private String certificate; + } + + @Data + public static class Decryption { + @JsonProperty(access = Access.WRITE_ONLY) + private String privateKey; + private String certificate; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class SecretInfo { + private String signingPrivateKey; + private String decryptionPrivateKey; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlRegistrationConfigHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlRegistrationConfigHelper.java new file mode 100644 index 0000000000..bc11acfcfe --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlRegistrationConfigHelper.java @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.integration.saml; + +import static com.oceanbase.odc.service.integration.util.EncryptionUtil.CERTIFICATE_KEY_PREFIX; +import static com.oceanbase.odc.service.integration.util.EncryptionUtil.CERTIFICATE_KEY_SUFFIX; +import static com.oceanbase.odc.service.integration.util.EncryptionUtil.PRIVATE_KEY_PREFIX; +import static com.oceanbase.odc.service.integration.util.EncryptionUtil.PRIVATE_KEY_SUFFIX; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.security.KeyFactory; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.security.interfaces.RSAPrivateKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.util.Collection; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import org.springframework.boot.context.properties.PropertyMapper; +import org.springframework.security.saml2.core.Saml2X509Credential; +import org.springframework.security.saml2.core.Saml2X509Credential.Saml2X509CredentialType; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistration; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistration.AssertingPartyDetails; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistration.Builder; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistrations; +import org.springframework.security.saml2.provider.service.registration.Saml2MessageBinding; +import org.springframework.util.StringUtils; + +import com.oceanbase.odc.common.util.EncodeUtils; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; +import com.oceanbase.odc.service.integration.saml.SamlParameter.Signing; + +public final class SamlRegistrationConfigHelper { + + public static RelyingPartyRegistration asRegistration(SSOIntegrationConfig ssoIntegrationConfig) { + Verify.verify("SAML".equals(ssoIntegrationConfig.getType()), "Invalid type=" + ssoIntegrationConfig.getType()); + SamlParameter parameter = (SamlParameter) ssoIntegrationConfig.getSsoParameter(); + boolean usingMetadata = StringUtils.hasText(parameter.getMetadataUri()); + Builder builder = (usingMetadata) + ? RelyingPartyRegistrations.fromMetadataLocation(parameter.getMetadataUri()) + .registrationId(parameter.getRegistrationId()) + : RelyingPartyRegistration.withRegistrationId(parameter.getRegistrationId()); + builder.assertionConsumerServiceLocation(parameter.getAcsLocation()); + builder.assertionConsumerServiceBinding(Saml2MessageBinding.valueOf(parameter.getAcsBinding())); + builder.assertingPartyDetails(mapAssertingParty(parameter, usingMetadata)); + builder.signingX509Credentials( + (credentials) -> addCredentialIfNotNull(credentials, () -> asSigningCredential(parameter))); + builder.decryptionX509Credentials( + (credentials) -> addCredentialIfNotNull(credentials, () -> asDecryptionCredential(parameter))); + builder.assertingPartyDetails((details) -> details + .verificationX509Credentials((credentials) -> addCredentialIfNotNull(credentials, + () -> asVerificationCredential(parameter)))); + builder.entityId(parameter.getAcsEntityId()); + RelyingPartyRegistration registration = builder.build(); + boolean signRequest = registration.getAssertingPartyDetails().getWantAuthnRequestsSigned(); + validateSigningCredentials(parameter, signRequest); + return registration; + } + + private static void addCredentialIfNotNull(Collection credentials, + Supplier supplier) { + Saml2X509Credential saml2X509Credential = supplier.get(); + if (saml2X509Credential != null) { + credentials.add(saml2X509Credential); + } + } + + private static Consumer mapAssertingParty(SamlParameter parameter, + boolean usingMetadata) { + return (details) -> { + PropertyMapper map = PropertyMapper.get().alwaysApplyingWhenNonNull(); + map.from(parameter::getProviderEntityId).to(details::entityId); + map.from(() -> Optional.ofNullable(parameter.getSinglesignon()) + .map(SamlParameter.Singlesignon::getBinding) + .map(Saml2MessageBinding::valueOf) + .orElse(null)) + .to(details::singleSignOnServiceBinding); + map.from(() -> Optional.ofNullable(parameter.getSinglesignon()) + .map(SamlParameter.Singlesignon::getUrl) + .orElse(null)) + .to(details::singleSignOnServiceLocation); + map.from(() -> Optional.ofNullable(parameter.getSinglesignon()) + .map(SamlParameter.Singlesignon::getSignRequest) + .orElse(null)) + .when((ignored) -> !usingMetadata) + .to(details::wantAuthnRequestsSigned); + }; + } + + private static Saml2X509Credential asSigningCredential(SamlParameter parameter) { + Signing signing = parameter.getSigning(); + if (signing == null || signing.getCertificate() == null) { + return null; + } + return asSaml2X509Credential(signing.getPrivateKey(), signing.getCertificate()); + } + + private static Saml2X509Credential asVerificationCredential(SamlParameter parameter) { + if (parameter.getVerification() == null || parameter.getVerification().getCertificate() == null) { + return null; + } + X509Certificate x509Certificate = getCertificateFromBase64(parameter.getVerification().getCertificate()); + return new Saml2X509Credential(x509Certificate, Saml2X509Credential.Saml2X509CredentialType.ENCRYPTION, + Saml2X509Credential.Saml2X509CredentialType.VERIFICATION); + } + + private static Saml2X509Credential asDecryptionCredential(SamlParameter parameter) { + if (parameter.getDecryption() == null || parameter.getDecryption().getCertificate() == null) { + return null; + } + Verify.notNull(parameter.getDecryption().getCertificate(), "certificate"); + Verify.notNull(parameter.getDecryption().getPrivateKey(), "privateKey"); + return asSaml2X509Credential(parameter.getDecryption().getPrivateKey(), + parameter.getDecryption().getCertificate()); + } + + private static Saml2X509Credential asSaml2X509Credential(String base64PrivateKey, String base64Certificate) { + RSAPrivateKey rsaPrivateKey = base64ToRSAPrivateKey(base64PrivateKey); + X509Certificate x509Certificate = getCertificateFromBase64(base64Certificate); + return new Saml2X509Credential(rsaPrivateKey, x509Certificate, Saml2X509CredentialType.SIGNING); + } + + private static RSAPrivateKey base64ToRSAPrivateKey(String base64PrivateKey) { + try { + byte[] privateKeyBytes = EncodeUtils.base64DecodeFromString(removeBase64PrivateKeyPem(base64PrivateKey)); + Verify.notNull(privateKeyBytes, "privateKeyBytes cannot be null"); + PKCS8EncodedKeySpec pkcs8EncodedKeySpec = new PKCS8EncodedKeySpec(privateKeyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + return (RSAPrivateKey) keyFactory.generatePrivate(pkcs8EncodedKeySpec); + } catch (Exception e) { + throw new RuntimeException("Invalid base64PrivateKey=" + base64PrivateKey, e); + } + } + + public static String removeBase64PrivateKeyPem(String base64PrivateKeyPem) { + return base64PrivateKeyPem + .replace(PRIVATE_KEY_PREFIX, "") + .replace(PRIVATE_KEY_SUFFIX, "") + .replaceAll("\\s", ""); + } + + private static X509Certificate getCertificateFromBase64(String base64Certificate) { + try { + byte[] certificateBytes = EncodeUtils.base64DecodeFromString(removeBase64CertificatePem(base64Certificate)); + CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509"); + Verify.notNull(certificateFactory, "certificateFactory cannot be null"); + InputStream in = new ByteArrayInputStream(certificateBytes); + return (X509Certificate) certificateFactory.generateCertificate(in); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid Base64 certificate=" + base64Certificate, e); + } + } + + public static String removeBase64CertificatePem(String base64Certificate) { + return base64Certificate + .replace(CERTIFICATE_KEY_PREFIX, "") + .replace(CERTIFICATE_KEY_SUFFIX, "") + .replace("\\n", "") + .replaceAll("\\s", ""); + } + + private static void validateSigningCredentials(SamlParameter parameter, boolean signRequest) { + if (signRequest) { + Verify.verify( + parameter.getSigning() != null && parameter.getSigning().getCertificate() != null + && parameter.getSigning().getPrivateKey() != null, + "Signing credentials must not be empty when authentication requests require signing."); + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/util/EncryptionUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/util/EncryptionUtil.java index cc98aa623c..5a9a1ceede 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/util/EncryptionUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/util/EncryptionUtil.java @@ -15,15 +15,38 @@ */ package com.oceanbase.odc.service.integration.util; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.SecureRandom; +import java.security.Security; +import java.security.cert.CertificateEncodingException; +import java.security.cert.X509Certificate; +import java.util.Date; import java.util.Objects; import java.util.concurrent.TimeUnit; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.KeyUsage; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; + import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; import com.oceanbase.odc.common.crypto.Encryptors; import com.oceanbase.odc.common.crypto.TextEncryptor; +import com.oceanbase.odc.common.lang.Pair; +import com.oceanbase.odc.common.util.EncodeUtils; import com.oceanbase.odc.service.integration.model.Encryption; +import lombok.SneakyThrows; + /** * Encryption utility for integration usage. * @@ -32,9 +55,19 @@ */ public class EncryptionUtil { + public static final String PRIVATE_KEY_PREFIX = "-----BEGIN PRIVATE KEY-----"; + public static final String PRIVATE_KEY_SUFFIX = "-----END PRIVATE KEY-----"; + public static final String CERTIFICATE_KEY_PREFIX = "-----BEGIN CERTIFICATE-----"; + public static final String CERTIFICATE_KEY_SUFFIX = "-----END CERTIFICATE-----"; private static final LoadingCache encryptorCache = Caffeine.newBuilder().maximumSize(100) .expireAfterAccess(10, TimeUnit.MINUTES).build(EncryptionUtil::getEncryptor); + static { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + public static String encrypt(String plainText, Encryption encryption) { return Objects.requireNonNull(encryptorCache.get(encryption)).encrypt(plainText); } @@ -57,4 +90,64 @@ private static TextEncryptor getEncryptor(Encryption encryption) { return Encryptors.empty(); } } + + @SneakyThrows + public static Pair generateKeyPair() { + KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA"); + keyPairGenerator.initialize(2048, new SecureRandom()); + KeyPair keyPair = keyPairGenerator.generateKeyPair(); + PrivateKey privateKey = keyPair.getPrivate(); + JcaX509v3CertificateBuilder certBuilder = getJcaX509v3CertificateBuilder(keyPair); + ContentSigner contentSigner = new JcaContentSignerBuilder("SHA256WithRSA").build(privateKey); + KeyUsage keyUsage = getKeyAllUsage(); + certBuilder.addExtension(Extension.keyUsage, true, keyUsage); + + X509Certificate certificate = new JcaX509CertificateConverter() + .setProvider(BouncyCastleProvider.PROVIDER_NAME) + .getCertificate(certBuilder.build(contentSigner)); + + return new Pair<>(privateKey, certificate); + } + + private static KeyUsage getKeyAllUsage() { + int keyUsageFlags = KeyUsage.digitalSignature + | KeyUsage.nonRepudiation + | KeyUsage.keyEncipherment + | KeyUsage.dataEncipherment + | KeyUsage.keyAgreement + | KeyUsage.keyCertSign + | KeyUsage.cRLSign + | KeyUsage.encipherOnly + | KeyUsage.decipherOnly; + return new KeyUsage(keyUsageFlags); + } + + private static JcaX509v3CertificateBuilder getJcaX509v3CertificateBuilder(KeyPair keyPair) { + PublicKey publicKey = keyPair.getPublic(); + + X500Name issuerName = new X500Name("CN=Test, O=ODC, L=Hangzhou, ST=Hangzhou, C=CN"); + BigInteger serialNumber = new BigInteger(64, new SecureRandom()); + Date startDate = new Date(); + Date endDate = new Date(startDate.getTime() + (365L * 24 * 60 * 60 * 1000)); + + return new JcaX509v3CertificateBuilder( + issuerName, serialNumber, startDate, endDate, issuerName, + publicKey); + } + + public static String convertPrivateKeyToPem(PrivateKey privateKey) { + byte[] encodedPrivateKey = privateKey.getEncoded(); + String base64EncodedKey = EncodeUtils.base64EncodeToString(encodedPrivateKey); + return PRIVATE_KEY_PREFIX + "\n" + + base64EncodedKey + + "\n" + PRIVATE_KEY_SUFFIX; + } + + public static String convertCertificateToPem(X509Certificate certificate) throws CertificateEncodingException { + byte[] encodedCertificate = certificate.getEncoded(); + String base64EncodedCert = EncodeUtils.base64EncodeToString(encodedCertificate); + return CERTIFICATE_KEY_PREFIX + "\n" + + base64EncodedCert + + "\n" + CERTIFICATE_KEY_SUFFIX; + } } diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/config/WebSecurityConfiguration.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/config/WebSecurityConfiguration.java index 8360ef658f..47d272a096 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/config/WebSecurityConfiguration.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/config/WebSecurityConfiguration.java @@ -18,6 +18,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import org.springframework.security.authentication.AuthenticationManager; @@ -26,6 +28,11 @@ import org.springframework.security.config.annotation.web.builders.WebSecurity; import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.saml2.provider.service.metadata.OpenSamlMetadataResolver; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistrationRepository; +import org.springframework.security.saml2.provider.service.web.DefaultRelyingPartyRegistrationResolver; +import org.springframework.security.saml2.provider.service.web.RelyingPartyRegistrationResolver; +import org.springframework.security.saml2.provider.service.web.Saml2MetadataFilter; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; import org.springframework.security.web.authentication.logout.LogoutSuccessHandler; import org.springframework.security.web.context.SecurityContextRepository; @@ -50,6 +57,9 @@ import com.oceanbase.odc.service.iam.auth.ldap.ODCLdapAuthenticator; import com.oceanbase.odc.service.iam.auth.local.LocalDaoAuthenticationProvider; import com.oceanbase.odc.service.iam.auth.oauth2.OAuth2SecurityConfigureHelper; +import com.oceanbase.odc.service.iam.auth.saml.CustomSamlProvider; +import com.oceanbase.odc.service.iam.auth.saml.DefaultSamlUserService; +import com.oceanbase.odc.service.iam.auth.saml.SamlSecurityConfigureHelper; import com.oceanbase.odc.service.iam.util.FailedLoginAttemptLimiter; import com.oceanbase.odc.service.integration.ldap.LdapConfigRegistrationManager; @@ -118,6 +128,14 @@ public class WebSecurityConfiguration extends WebSecurityConfigurerAdapter { @Autowired private LdapConfigRegistrationManager ldapConfigRegistrationManager; + @Autowired + private RelyingPartyRegistrationRepository registrations; + + @Autowired + private DefaultSamlUserService defaultSamlUserService; + + @Autowired + private SamlSecurityConfigureHelper samlSecurityConfigureHelper; private BastionAuthenticationProvider bastionAuthenticationProvider() { return new BastionAuthenticationProvider(bastionUserDetailService); @@ -127,6 +145,7 @@ private BastionAuthenticationProvider bastionAuthenticationProvider() { public void configure(AuthenticationManagerBuilder auth) { auth.authenticationProvider(localDaoAuthenticationProvider) .authenticationProvider(bastionAuthenticationProvider()) + .authenticationProvider(new CustomSamlProvider(defaultSamlUserService)) .authenticationProvider( new ODCLdapAuthenticationProvider(new ODCLdapAuthenticator(ldapConfigRegistrationManager), ldapUserDetailsContextMapper)); @@ -139,11 +158,28 @@ public void configure(WebSecurity web) { .and().ignoring().antMatchers(commonSecurityProperties.getAuthWhitelist()); } + @Bean + RelyingPartyRegistrationResolver relyingPartyRegistrationResolver( + RelyingPartyRegistrationRepository registrations) { + return new DefaultRelyingPartyRegistrationResolver(registrations); + } + + @Bean + FilterRegistrationBean metadata(RelyingPartyRegistrationResolver registrations) { + Saml2MetadataFilter metadata = new Saml2MetadataFilter(registrations, new OpenSamlMetadataResolver()); + FilterRegistrationBean filter = new FilterRegistrationBean<>(metadata); + filter.setOrder(-101); + return filter; + } + + @Override protected void configure(HttpSecurity http) throws Exception { corsConfigureHelper.configure(http); usernamePasswordConfigureHelper.configure(http, authenticationManager()); oauth2SecurityConfigureHelper.configure(http); + samlSecurityConfigureHelper.configure(http, authenticationManager()); + ldapSecurityConfigureHelper.configure(http, authenticationManager()); SecurityContextRepository securityContextRepository = securityContextRepository(); diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/MappingRuleConvert.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/MappingRuleConvert.java index fd812b904f..21a28c838a 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/MappingRuleConvert.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/MappingRuleConvert.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nullable; import javax.naming.NamingEnumeration; @@ -35,6 +36,8 @@ import org.springframework.ldap.core.DirContextOperations; import org.springframework.security.authentication.InternalAuthenticationServiceException; import org.springframework.security.oauth2.client.userinfo.OAuth2UserRequest; +import org.springframework.security.saml2.provider.service.authentication.DefaultSaml2AuthenticatedPrincipal; +import org.springframework.security.saml2.provider.service.authentication.Saml2Authentication; import org.springframework.stereotype.Component; import com.google.common.base.MoreObjects; @@ -51,6 +54,7 @@ import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig.MappingRule; import com.oceanbase.odc.service.integration.oauth2.AddableClientRegistrationManager; import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; +import com.oceanbase.odc.service.integration.saml.AddableRelyingPartyRegistrationRepository; import lombok.NonNull; import lombok.SneakyThrows; @@ -69,11 +73,14 @@ public class MappingRuleConvert { @Autowired private TestLoginManager testLoginManager; + @Autowired + private AddableRelyingPartyRegistrationRepository addableRelyingPartyRegistrationRepository; + public MappingResult resolveOAuthMappingResult(OAuth2UserRequest userRequest, Map userInfoMap) { MappingRule mappingRule = resolveMappingRule(userRequest); Verify.notNull(mappingRule, "mappingRule"); userInfoMap = getUserInfoMapFromResponse(userInfoMap, mappingRule); - testLoginManager.saveOauth2TestIdIfNeed(JsonUtils.toJson(userInfoMap)); + testLoginManager.saveOauth2InfoIfNeed(JsonUtils.toJson(userInfoMap)); testLoginManager.abortIfOAuthTestLoginInfo(); Long organizationId = parseOrganizationId(userRequest.getClientRegistration().getRegistrationId()); String userAccountName = String.valueOf(userInfoMap.get(mappingRule.getUserAccountNameField())); @@ -110,6 +117,40 @@ public MappingResult resolveLdapMappingResult(DirContextOperations ctx, String u .build(); } + public MappingResult resolveSamlMappingResult(Saml2Authentication saml2Authentication) { + DefaultSaml2AuthenticatedPrincipal principal = + (DefaultSaml2AuthenticatedPrincipal) saml2Authentication.getPrincipal(); + Map userInfoMap = getUserInfoMap(principal); + testLoginManager.saveSamlInfoIfNeed(JsonUtils.toJson(userInfoMap)); + testLoginManager.abortIfSamlTestLogin(); + + String relyingPartyRegistrationId = principal.getRelyingPartyRegistrationId(); + + SSOIntegrationConfig ssoIntegrationConfig = + addableRelyingPartyRegistrationRepository.findConfigByRegistrationId(relyingPartyRegistrationId); + com.google.common.base.Verify.verifyNotNull(ssoIntegrationConfig, "ssoIntegrationConfig"); + MappingRule mappingRule = ssoIntegrationConfig.getMappingRule(); + String name = principal.getName(); + String parseExtraInfo = parseExtraInfo(userInfoMap, mappingRule); + + return MappingResult.builder() + .organizationId(SSOIntegrationConfig.parseOrganizationId(relyingPartyRegistrationId)) + .userAccountName(name) + .userNickName(getName(userInfoMap, mappingRule)) + .isAdmin(false) + .extraInfo(parseExtraInfo) + .sourceUserInfoMap(userInfoMap) + .build(); + } + + private Map getUserInfoMap(DefaultSaml2AuthenticatedPrincipal principal) { + return principal.getAttributes().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> entry.getValue().size() == 1 ? entry.getValue().get(0) : entry.getValue())); + } + + @SneakyThrows private Map getUserInfoMap(DirContextOperations ctx) { NamingEnumeration all = ctx.getAttributes().getAll(); @@ -149,9 +190,9 @@ private MappingRule resolveMappingRule(OAuth2UserRequest userRequest) { @Nullable - private String getName(Map oAuth2User, MappingRule mappingRule) { + private String getName(Map userInfoMap, MappingRule mappingRule) { Set nameFields = MoreObjects.firstNonNull(mappingRule.getUserNickNameField(), new HashSet<>()); - Object o = nameFields.stream().map(oAuth2User::get).filter(Objects::nonNull).findFirst().orElse(null); + Object o = nameFields.stream().map(userInfoMap::get).filter(Objects::nonNull).findFirst().orElse(null); return o == null ? null : String.valueOf(o); } diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/CustomOAuth2AuthorizationRequestResolver.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/CustomOAuth2AuthorizationRequestResolver.java index 661d759e93..588ba09747 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/CustomOAuth2AuthorizationRequestResolver.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/CustomOAuth2AuthorizationRequestResolver.java @@ -33,7 +33,7 @@ import com.oceanbase.odc.service.integration.model.Oauth2Parameter; import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; import com.oceanbase.odc.service.integration.oauth2.AddableClientRegistrationManager; -import com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import com.oceanbase.odc.service.state.StatefulUuidStateIdGenerator; import lombok.SneakyThrows; @@ -44,14 +44,14 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza private final AddableClientRegistrationManager addableClientRegistrationManager; private StatefulUuidStateIdGenerator statefulUuidStateIdGenerator; - private Oauth2StateManager oauth2StateManager; + private SSOStateManager sSOStateManager; private AntPathRequestMatcher authorizationRequestMatcher; public CustomOAuth2AuthorizationRequestResolver(AddableClientRegistrationManager addableClientRegistrationManager, - StatefulUuidStateIdGenerator statefulUuidStateIdGenerator, Oauth2StateManager oauth2StateManager) { + StatefulUuidStateIdGenerator statefulUuidStateIdGenerator, SSOStateManager sSOStateManager) { this.addableClientRegistrationManager = addableClientRegistrationManager; this.statefulUuidStateIdGenerator = statefulUuidStateIdGenerator; - this.oauth2StateManager = oauth2StateManager; + this.sSOStateManager = sSOStateManager; DefaultOAuth2AuthorizationRequestResolver defaultOAuth2AuthorizationRequestResolver = new DefaultOAuth2AuthorizationRequestResolver( addableClientRegistrationManager, "/oauth2/authorization"); @@ -95,10 +95,10 @@ private OAuth2AuthorizationRequest doResolve(HttpServletRequest request, String String state = authorizationRequest.getState(); String originRedirectUrl = authorizationRequest.getRedirectUri(); UriComponentsBuilder.fromUriString(originRedirectUrl).build().getQueryParams() - .forEach((key, value) -> oauth2StateManager.setStateParameter(state, key, urlDecode(value.get(0)))); + .forEach((key, value) -> sSOStateManager.setStateParameter(state, key, urlDecode(value.get(0)))); URL url = new URL(originRedirectUrl); String urlWithoutQuery = new URL(url.getProtocol(), url.getHost(), url.getPort(), url.getPath()).toString(); - oauth2StateManager.setOdcParameters(state, request); + sSOStateManager.setOdcParameters(state, request); return OAuth2AuthorizationRequest.from(authorizationRequest) .redirectUri(urlWithoutQuery) diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2SecurityConfigureHelper.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2SecurityConfigureHelper.java index ea0fa20787..831ceec0cc 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2SecurityConfigureHelper.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2SecurityConfigureHelper.java @@ -29,7 +29,7 @@ import com.oceanbase.odc.service.iam.auth.CustomAuthenticationFailureHandler; import com.oceanbase.odc.service.iam.auth.CustomAuthenticationSuccessHandler; import com.oceanbase.odc.service.integration.oauth2.AddableClientRegistrationManager; -import com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import com.oceanbase.odc.service.state.StatefulUuidStateIdGenerator; @Component @@ -54,7 +54,7 @@ public class OAuth2SecurityConfigureHelper { @Autowired private StatefulUuidStateIdGenerator statefulUuidStateIdGenerator; @Autowired - private Oauth2StateManager oauth2StateManager; + private SSOStateManager SSOStateManager; public void configure(HttpSecurity http) @@ -65,7 +65,7 @@ public void configure(HttpSecurity http) .authorizationEndpoint() .authorizationRequestResolver( new CustomOAuth2AuthorizationRequestResolver(this.addableClientRegistrationManager, - statefulUuidStateIdGenerator, oauth2StateManager)) + statefulUuidStateIdGenerator, SSOStateManager)) .and() // token 端点配置, 根据 code 获取 token .tokenEndpoint() @@ -78,7 +78,7 @@ public void configure(HttpSecurity http) .oidcUserService(oidcUserService); http.addFilterBefore( - new OAuth2AbstractTestLoginAuthenticationFilter(), + new OAuth2TestLoginAuthenticationFilter(), OAuth2LoginAuthenticationFilter.class); } } diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2AbstractTestLoginAuthenticationFilter.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2TestLoginAuthenticationFilter.java similarity index 90% rename from server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2AbstractTestLoginAuthenticationFilter.java rename to server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2TestLoginAuthenticationFilter.java index a4f10f0d0d..1eae4b730b 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2AbstractTestLoginAuthenticationFilter.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2TestLoginAuthenticationFilter.java @@ -20,7 +20,7 @@ import com.oceanbase.odc.service.iam.auth.local.AbstractTestLoginAuthenticationFilter; import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; -public class OAuth2AbstractTestLoginAuthenticationFilter extends AbstractTestLoginAuthenticationFilter { +public class OAuth2TestLoginAuthenticationFilter extends AbstractTestLoginAuthenticationFilter { @Override protected Boolean isTestRequest(HttpServletRequest request) { diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2UserServiceImpl.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2UserServiceImpl.java index a384181317..cb8aae90ed 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2UserServiceImpl.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2UserServiceImpl.java @@ -36,6 +36,7 @@ import org.springframework.security.oauth2.core.OAuth2AuthenticationException; import org.springframework.security.oauth2.core.OAuth2AuthorizationException; import org.springframework.security.oauth2.core.OAuth2Error; +import org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames; import org.springframework.security.oauth2.core.user.OAuth2User; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; @@ -49,7 +50,7 @@ import com.oceanbase.odc.service.iam.auth.MappingRuleConvert; import com.oceanbase.odc.service.iam.auth.SsoUserDetailService; import com.oceanbase.odc.service.iam.model.User; -import com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; /** @@ -80,7 +81,7 @@ public class OAuth2UserServiceImpl implements OAuth2UserService> response = getResponse(userRequest, request); PreConditions.notNull(response, "oAuth2User"); - oauth2StateManager.addStateToCurrentRequestParam(); + ssoStateManager.addStateToCurrentRequestParam(OAuth2ParameterNames.STATE); MappingResult mappingResult = mappingRuleConvert.resolveOAuthMappingResult(userRequest, response.getBody()); testLoginManager.abortIfOAuthTestLoginTest(); diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OidcUserServiceImpl.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OidcUserServiceImpl.java index fa865c22f8..7eb3618124 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OidcUserServiceImpl.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OidcUserServiceImpl.java @@ -40,6 +40,7 @@ import org.springframework.security.oauth2.core.OAuth2Error; import org.springframework.security.oauth2.core.converter.ClaimConversionService; import org.springframework.security.oauth2.core.converter.ClaimTypeConverter; +import org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames; import org.springframework.security.oauth2.core.oidc.OidcIdToken; import org.springframework.security.oauth2.core.oidc.OidcScopes; import org.springframework.security.oauth2.core.oidc.OidcUserInfo; @@ -56,7 +57,7 @@ import com.oceanbase.odc.service.iam.auth.MappingRuleConvert; import com.oceanbase.odc.service.iam.auth.SsoUserDetailService; import com.oceanbase.odc.service.iam.model.User; -import com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; /** @@ -88,7 +89,7 @@ public class OidcUserServiceImpl implements OAuth2UserService> createDefaultClaimTypeConverters() { Converter booleanConverter = getConverter(TypeDescriptor.valueOf(Boolean.class)); @@ -135,7 +136,7 @@ public User loadUser(OidcUserRequest userRequest) throws OAuth2AuthenticationExc } } Map userInfoMap = collectClaims(userRequest.getIdToken(), userInfo); - oauth2StateManager.addStateToCurrentRequestParam(); + SSOStateManager.addStateToCurrentRequestParam(OAuth2ParameterNames.STATE); MappingResult mappingResult = mappingRuleConvert.resolveOAuthMappingResult(userRequest, userInfoMap); testLoginManager.abortIfOAuthTestLoginTest(); diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/CustomSamlProvider.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/CustomSamlProvider.java new file mode 100644 index 0000000000..ca826a4e3d --- /dev/null +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/CustomSamlProvider.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.auth.saml; + +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.saml2.provider.service.authentication.OpenSamlAuthenticationProvider; +import org.springframework.security.saml2.provider.service.authentication.Saml2Authentication; + +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.service.iam.model.User; + +/** + * why ues OpenSamlAuthenticationProvider instead of OpenSaml4AuthenticationProvider ? according to + * {@link "https://github.com/spring-projects/spring-security/issues/11434"}, + * OpenSaml4AuthenticationProvider depends on opensaml 4.1+, which requires jdk11. In the current + * version is loaded by default implementation OpenSaml4AuthenticationProvider, although it has been + * deprecated. + */ +public class CustomSamlProvider implements AuthenticationProvider { + + private final AuthenticationProvider defaultAuthenticationProvider; + + private final DefaultSamlUserService defaultSamlUserService; + + public CustomSamlProvider(DefaultSamlUserService defaultSamlUserService) { + this.defaultSamlUserService = defaultSamlUserService; + defaultAuthenticationProvider = new OpenSamlAuthenticationProvider(); + } + + @Override + public Authentication authenticate(Authentication authentication) throws AuthenticationException { + Authentication authenticate = defaultAuthenticationProvider.authenticate(authentication); + Verify.verify(authenticate instanceof Saml2Authentication, + "invalid type of authentication, class: " + authentication.getClass()); + Saml2Authentication saml2Authentication = (Saml2Authentication) authenticate; + User user = defaultSamlUserService.loadUser(saml2Authentication); + return new Saml2Authentication(user, ((Saml2Authentication) authenticate).getSaml2Response(), + authenticate.getAuthorities()); + } + + @Override + public boolean supports(Class authentication) { + return defaultAuthenticationProvider.supports(authentication); + } +} diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/DefaultSamlUserService.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/DefaultSamlUserService.java new file mode 100644 index 0000000000..7c7bf0ac5b --- /dev/null +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/DefaultSamlUserService.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.auth.saml; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.saml2.provider.service.authentication.Saml2Authentication; +import org.springframework.stereotype.Service; + +import com.oceanbase.odc.service.iam.auth.MappingRuleConvert; +import com.oceanbase.odc.service.iam.auth.SsoUserDetailService; +import com.oceanbase.odc.service.iam.auth.oauth2.MappingResult; +import com.oceanbase.odc.service.iam.model.User; + +@Service +public class DefaultSamlUserService { + + @Autowired + private SsoUserDetailService ssoUserDetailService; + + @Autowired + private MappingRuleConvert mappingRuleConvert; + + public User loadUser(Saml2Authentication saml2Authentication) { + MappingResult mappingResult = mappingRuleConvert.resolveSamlMappingResult(saml2Authentication); + return ssoUserDetailService.getOrCreateUser(mappingResult); + } + + +} diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlSecurityConfigureHelper.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlSecurityConfigureHelper.java new file mode 100644 index 0000000000..2690de072c --- /dev/null +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlSecurityConfigureHelper.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.auth.saml; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Profile; +import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistrationRepository; +import org.springframework.security.saml2.provider.service.servlet.filter.Saml2WebSsoAuthenticationFilter; +import org.springframework.stereotype.Component; + +import com.oceanbase.odc.service.iam.auth.CustomAuthenticationFailureHandler; +import com.oceanbase.odc.service.iam.auth.CustomAuthenticationSuccessHandler; + +@Component +@Profile("alipay") +@ConditionalOnProperty(value = {"odc.iam.auth.type"}, havingValue = "local") +public class SamlSecurityConfigureHelper { + + + @Autowired + private RelyingPartyRegistrationRepository registrations; + + @Autowired + private CustomAuthenticationSuccessHandler customAuthenticationSuccessHandler; + + @Autowired + private CustomAuthenticationFailureHandler customAuthenticationFailureHandler; + + public void configure(HttpSecurity http, AuthenticationManager authenticationManager) + throws Exception { + http.saml2Login(); + Saml2WebSsoAuthenticationFilter saml2WebSsoAuthenticationFilter = + new Saml2WebSsoAuthenticationFilter(registrations); + saml2WebSsoAuthenticationFilter.setAuthenticationSuccessHandler(customAuthenticationSuccessHandler); + saml2WebSsoAuthenticationFilter.setAuthenticationFailureHandler(customAuthenticationFailureHandler); + saml2WebSsoAuthenticationFilter.setAuthenticationManager(authenticationManager); + http.addFilterBefore( + saml2WebSsoAuthenticationFilter, + Saml2WebSsoAuthenticationFilter.class); + http.addFilterBefore( + new SamlTestLoginAuthenticationFilter(), + Saml2WebSsoAuthenticationFilter.class); + } +} diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlTestLoginAuthenticationFilter.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlTestLoginAuthenticationFilter.java new file mode 100644 index 0000000000..e82de6f241 --- /dev/null +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlTestLoginAuthenticationFilter.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.auth.saml; + +import javax.servlet.http.HttpServletRequest; + +import com.oceanbase.odc.service.iam.auth.local.AbstractTestLoginAuthenticationFilter; +import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; + +public class SamlTestLoginAuthenticationFilter extends AbstractTestLoginAuthenticationFilter { + @Override + protected Boolean isTestRequest(HttpServletRequest request) { + return TestLoginManager.isSamlTestLoginRequest(request); + } +} diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/info/WebInfoAdapter.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/info/WebInfoAdapter.java index 2473cb79f7..e044107b6a 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/info/WebInfoAdapter.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/info/WebInfoAdapter.java @@ -39,17 +39,17 @@ @SkipAuthorize("odc internal usage") public class WebInfoAdapter implements InfoAdapter { - @Value("${odc.iam.password-login-enabled:true}") - private Boolean passwordLoginEnabled; @Value("${odc.iam.auth.type}") protected Set authType; + @Autowired + protected IntegrationService integrationService; + @Value("${odc.iam.password-login-enabled:true}") + private Boolean passwordLoginEnabled; @Value("${odc.help.supportGroupQRCodeUrl:#{null}}") private String supportGroupQRCodeUrl; @Autowired private PlaysiteOpenApiProperties alipayOpenApiProperties; @Autowired - protected IntegrationService integrationService; - @Autowired private BuildProperties buildProperties; @Override @@ -66,7 +66,7 @@ public String getLoginUrl(HttpServletRequest request) { return alipayOpenApiProperties.getObOfficialLoginUrl(); } else if (authType.contains("local")) { SSOIntegrationConfig sSoClientRegistration = integrationService.getSSoIntegrationConfig(); - if (sSoClientRegistration == null || !sSoClientRegistration.isOauth2OrOidc()) { + if (sSoClientRegistration == null) { return null; } return sSoClientRegistration.resolveLoginRedirectUrl(); From 525ff22cb55540abfd34c4bbdff97ff8c41d5fe1 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Fri, 6 Dec 2024 14:33:00 +0800 Subject: [PATCH 046/118] fix(migrate): rename V_4_3_3_1 to V_4_3_3_2 (#3974) --- ...index.sql => V_4_3_3_2__alter_resource_resource_add_index.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/odc-migrate/src/main/resources/migrate/common/{V_4_3_3_1__alter_resource_resource_add_index.sql => V_4_3_3_2__alter_resource_resource_add_index.sql} (100%) diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_resource_resource_add_index.sql b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_2__alter_resource_resource_add_index.sql similarity index 100% rename from server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_resource_resource_add_index.sql rename to server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_2__alter_resource_resource_add_index.sql From f406504d96f0d1f0526ff0a47e810ab2f5dc147e Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Fri, 6 Dec 2024 14:38:46 +0800 Subject: [PATCH 047/118] feat(permission): add view permission control (#3946) * Add view support and optimize table sync in TableService * Add support for VIEW and EXTERNAL_TABLE in ApplyTablePermissionPreprocessor * Optimize view support in TableService * modify code format * add unit tests * rename method * modify code format * Remove redundant assertions in TableServiceTest * pass unit tests * Refactor DBObjectRepository and enhance permission handling * Grant SYSDBA role to test user in TestDBConfigurations --- .../odc/service/db/TableServiceTest.java | 118 ++++++++++-------- .../metadb/dbobject/DBObjectRepository.java | 2 +- .../connection/table/TableService.java | 34 ++++- .../DBResourcePermissionHelper.java | 4 +- .../ApplyTablePermissionPreprocessor.java | 3 +- .../test/database/TestDBConfigurations.java | 5 + 6 files changed, 103 insertions(+), 63 deletions(-) diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/TableServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/TableServiceTest.java index 639cb63721..23ee2c0949 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/TableServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/TableServiceTest.java @@ -15,10 +15,11 @@ */ package com.oceanbase.odc.service.db; +import static org.mockito.ArgumentMatchers.eq; + import java.sql.SQLException; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.stream.Collectors; @@ -71,15 +72,18 @@ public class TableServiceTest extends ServiceTestEnv { private DBResourcePermissionHelper dbResourcePermissionHelper; private static final String OB_MYSQL_TABLE_CREATE_TEMPLATE = "CREATE TABLE IF NOT EXISTS %s (\n" - + " id BIGINT NOT NULL AUTO_INCREMENT,\n" - + " PRIMARY key (`id`)\n" + + " ID BIGINT NOT NULL AUTO_INCREMENT,\n" + + " PRIMARY KEY (`ID`)\n" + ")"; private static final String OB_ORACLE_TABLE_CREATE_TEMPLATE = "CREATE TABLE %s (\n" - + " id NUMBER PRIMARY KEY\n" + + " ID NUMBER PRIMARY KEY\n" + ")"; - - private final static List TABLE_NAME_LIST = Arrays.asList("test_table_1", "test_table_2", "test_table_3"); + private static final String TABLE_DROP_TEMPLATE = "DROP TABLE %s"; + private static final String VIEW_CREATE_TEMPLATE = "CREATE VIEW %s AS SELECT 1 AS dummy FROM dual"; + private static final String VIEW_DROP_TEMPLATE = "DROP VIEW %s"; + private final static List TABLE_NAME_LIST = Arrays.asList("TEST_TABLE_1", "TEST_TABLE_2", "TEST_TABLE_3"); + private final static List VIEW_NAME_LIST = Arrays.asList("TEST_VIEW_10", "TEST_VIEW_20", "TEST_VIEW_30"); private final static Long DATABASE_ID = 1L; @@ -87,18 +91,27 @@ public class TableServiceTest extends ServiceTestEnv { @BeforeClass public static void setUp() { - createTablesByConnectType(ConnectType.OB_MYSQL, TABLE_NAME_LIST, OB_MYSQL_TABLE_CREATE_TEMPLATE); - createTablesByConnectType(ConnectType.OB_ORACLE, TABLE_NAME_LIST, OB_ORACLE_TABLE_CREATE_TEMPLATE); - createTablesByConnectType(ConnectType.MYSQL, TABLE_NAME_LIST, OB_MYSQL_TABLE_CREATE_TEMPLATE); - createTablesByConnectType(ConnectType.ORACLE, TABLE_NAME_LIST, OB_ORACLE_TABLE_CREATE_TEMPLATE); + clear(); + createTablesOrViewsByConnectType(ConnectType.OB_MYSQL, TABLE_NAME_LIST, OB_MYSQL_TABLE_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.OB_ORACLE, TABLE_NAME_LIST, OB_ORACLE_TABLE_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.MYSQL, TABLE_NAME_LIST, OB_MYSQL_TABLE_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.ORACLE, TABLE_NAME_LIST, OB_ORACLE_TABLE_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.OB_MYSQL, VIEW_NAME_LIST, VIEW_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.OB_ORACLE, VIEW_NAME_LIST, VIEW_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.MYSQL, VIEW_NAME_LIST, VIEW_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.ORACLE, VIEW_NAME_LIST, VIEW_CREATE_TEMPLATE); } @AfterClass public static void clear() { - dropTablesByConnectTypes(ConnectType.OB_MYSQL, TABLE_NAME_LIST); - dropTablesByConnectTypes(ConnectType.OB_ORACLE, TABLE_NAME_LIST); - dropTablesByConnectTypes(ConnectType.MYSQL, TABLE_NAME_LIST); - dropTablesByConnectTypes(ConnectType.ORACLE, TABLE_NAME_LIST); + dropTablesOrViewsByConnectTypes(ConnectType.OB_MYSQL, VIEW_NAME_LIST, VIEW_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.OB_ORACLE, VIEW_NAME_LIST, VIEW_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.MYSQL, VIEW_NAME_LIST, VIEW_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.ORACLE, VIEW_NAME_LIST, VIEW_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.OB_MYSQL, TABLE_NAME_LIST, TABLE_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.OB_ORACLE, TABLE_NAME_LIST, TABLE_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.MYSQL, TABLE_NAME_LIST, TABLE_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.ORACLE, TABLE_NAME_LIST, TABLE_DROP_TEMPLATE); } @Test @@ -161,14 +174,15 @@ public void list_whenConnectionTypeIsOracleAndOrganizationTypeIsTeam_succeed() testByConnectionTypeInTeamSpace(ConnectType.ORACLE); } - private List getTableEntities() { - return TABLE_NAME_LIST.stream().map(name -> { + private List getTableEntities(DBObjectType dbObjectType, List names) { + return names.stream().map(name -> { DBObjectEntity entity = new DBObjectEntity(); entity.setName(name); - entity.setType(DBObjectType.TABLE); + entity.setType(dbObjectType); entity.setDatabaseId(DATABASE_ID); return entity; }).collect(java.util.stream.Collectors.toList()); + } private User getIndivisualUser() { @@ -194,23 +208,8 @@ private Database getDatabaseByConnectType(ConnectType connectType) { return database; } - private boolean containsAllIgnoreCase(List list, List toCheck) { - for (String item : toCheck) { - boolean found = false; - for (String listItem : list) { - if (listItem.equalsIgnoreCase(item)) { - found = true; - break; - } - } - if (!found) { - return false; - } - } - return true; - } - - private static void createTablesByConnectType(ConnectType connectType, List tableNames, String format) { + private static void createTablesOrViewsByConnectType(ConnectType connectType, List tableNames, + String format) { ConnectionSession session = TestConnectionUtil.getTestConnectionSession(connectType); SyncJdbcExecutor syncJdbcExecutor = session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); for (String name : tableNames) { @@ -218,45 +217,56 @@ private static void createTablesByConnectType(ConnectType connectType, List tableNames) { + private static void dropTablesOrViewsByConnectTypes(ConnectType connectType, List tableNames, + String format) { ConnectionSession connectionSession = TestConnectionUtil.getTestConnectionSession(connectType); JdbcOperations jdbcOperations = connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); for (String name : tableNames) { - jdbcOperations.execute(String.format("drop table %s", name)); + try { + jdbcOperations.execute(String.format(format, name)); + } catch (Exception e) { + // ignore + } } } private void testByConnectionTypeInIndividualSpace(ConnectType connectType) throws SQLException, InterruptedException { Mockito.when(authenticationFacade.currentUser()).thenReturn(getIndivisualUser()); - Mockito.when(databaseService.detail(Mockito.any())).thenReturn(getDatabaseByConnectType(connectType)); - QueryTableParams params = QueryTableParams.builder() - .databaseId(DATABASE_ID) - .types(Collections.singletonList(DBObjectType.TABLE)) - .includePermittedAction(false) - .build(); - List
list = tableService.list(params); - Assert.assertFalse(list.isEmpty()); - List nameList = list.stream().map(Table::getName).collect(Collectors.toList()); - Assert.assertTrue(containsAllIgnoreCase(nameList, TABLE_NAME_LIST)); + testByConnectionType(connectType); } - private void testByConnectionTypeInTeamSpace(ConnectType obMysql) throws SQLException, InterruptedException { + private void testByConnectionTypeInTeamSpace(ConnectType connectType) throws SQLException, InterruptedException { Mockito.when(authenticationFacade.currentUser()).thenReturn(getTeamUser()); - Mockito.when(databaseService.detail(Mockito.any())).thenReturn(getDatabaseByConnectType(obMysql)); - Mockito.when(dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(Mockito.anyLong(), Mockito.any())) - .thenReturn(getTableEntities()); - Mockito.when(dbResourcePermissionHelper.getTablePermissions(Mockito.any())).thenReturn(new HashMap<>()); + Mockito.when( + dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(Mockito.anyLong(), eq(DBObjectType.TABLE))) + .thenReturn(getTableEntities(DBObjectType.TABLE, TABLE_NAME_LIST)); + Mockito.when(dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(Mockito.anyLong(), eq(DBObjectType.VIEW))) + .thenReturn(getTableEntities(DBObjectType.VIEW, VIEW_NAME_LIST)); + Mockito.when(dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(Mockito.anyLong(), + eq(DBObjectType.EXTERNAL_TABLE))) + .thenReturn(Arrays.asList()); + testByConnectionType(connectType); + } + + private void testByConnectionType(ConnectType connectType) throws SQLException, InterruptedException { + Mockito.when(databaseService.detail(Mockito.any())).thenReturn(getDatabaseByConnectType(connectType)); QueryTableParams params = QueryTableParams.builder() .databaseId(DATABASE_ID) - .types(Collections.singletonList(DBObjectType.TABLE)) + .types(Arrays.asList(DBObjectType.TABLE, DBObjectType.VIEW, DBObjectType.EXTERNAL_TABLE)) .includePermittedAction(false) .build(); List
list = tableService.list(params); Assert.assertFalse(list.isEmpty()); - List nameList = list.stream().map(Table::getName).collect(Collectors.toList()); - Assert.assertTrue(containsAllIgnoreCase(nameList, TABLE_NAME_LIST)); + List tableList = list.stream().filter(table -> table.getType() == DBObjectType.TABLE).map( + Table::getName).map(String::toUpperCase).collect( + Collectors.toList()); + List viewList = list.stream().filter(table -> table.getType() == DBObjectType.VIEW).map(Table::getName) + .map(String::toUpperCase).collect( + Collectors.toList()); + Assert.assertTrue(tableList.containsAll(TABLE_NAME_LIST)); + Assert.assertTrue(viewList.containsAll(VIEW_NAME_LIST)); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java index 1f8fcc874f..90be41f7f4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java @@ -43,7 +43,7 @@ public interface DBObjectRepository extends OdcJpaRepository findByDatabaseIdAndTypeIn(Long databaseId, Collection types); - List findByDatabaseIdInAndType(Collection databaseIds, DBObjectType type); + List findByDatabaseIdInAndTypeIn(Collection databaseIds, Collection types); /** * list physical tables that not belongs to any logical tables in a physical database diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java index 0e3eef06b8..0cf613ec4a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java @@ -51,6 +51,7 @@ import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint; import com.oceanbase.odc.plugin.schema.api.TableExtensionPoint; +import com.oceanbase.odc.plugin.schema.api.ViewExtensionPoint; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; @@ -60,6 +61,7 @@ import com.oceanbase.odc.service.db.schema.syncer.DBSchemaSyncer; import com.oceanbase.odc.service.db.schema.syncer.object.DBExternalTableSyncer; import com.oceanbase.odc.service.db.schema.syncer.object.DBTableSyncer; +import com.oceanbase.odc.service.db.schema.syncer.object.DBViewSyncer; import com.oceanbase.odc.service.feature.VersionDiffConfigService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; @@ -100,6 +102,9 @@ public class TableService { @Autowired private DBExternalTableSyncer dbExternalTableSyncer; + @Autowired + private DBViewSyncer dbViewSyncer; + @Autowired private JdbcLockRegistry lockRegistry; @@ -128,8 +133,10 @@ public List
list(@NonNull @Valid QueryTableParams params) throws SQLExcep + " doesn't support the database object type " + DBObjectType.TABLE); } if (types.contains(DBObjectType.TABLE)) { + Set latestTableNames = tableExtension.list(conn, database.getName(), DBObjectType.TABLE) + .stream().map(DBObjectIdentity::getName).collect(Collectors.toCollection(LinkedHashSet::new)); generateListAndSyncDBTablesByTableType(params, database, dataSource, tables, conn, DBObjectType.TABLE, - tableExtension); + latestTableNames); } if (types.contains(DBObjectType.EXTERNAL_TABLE)) { InformationExtensionPoint point = @@ -137,8 +144,23 @@ public List
list(@NonNull @Valid QueryTableParams params) throws SQLExcep String databaseProductVersion = point.getDBVersion(conn); if (versionDiffConfigService.isExternalTableSupported(dataSource.getDialectType(), databaseProductVersion)) { + Set latestExternalTableNames = + tableExtension.list(conn, database.getName(), DBObjectType.EXTERNAL_TABLE) + .stream().map(DBObjectIdentity::getName) + .collect(Collectors.toCollection(LinkedHashSet::new)); + generateListAndSyncDBTablesByTableType(params, database, dataSource, tables, conn, + DBObjectType.EXTERNAL_TABLE, latestExternalTableNames); + } + } + if (types.contains(DBObjectType.VIEW)) { + ViewExtensionPoint viewExtension = SchemaPluginUtil.getViewExtension(dataSource.getDialectType()); + if (viewExtension != null) { + Set latestViewNames = viewExtension.list(conn, database.getName()) + .stream().map(DBObjectIdentity::getName) + .collect(Collectors.toCollection(LinkedHashSet::new)); generateListAndSyncDBTablesByTableType(params, database, dataSource, tables, conn, - DBObjectType.EXTERNAL_TABLE, tableExtension); + DBObjectType.VIEW, + latestViewNames); } } } @@ -146,10 +168,8 @@ public List
list(@NonNull @Valid QueryTableParams params) throws SQLExcep } private void generateListAndSyncDBTablesByTableType(QueryTableParams params, Database database, - ConnectionConfig dataSource, List
tables, Connection conn, DBObjectType tableType, - TableExtensionPoint tableExtension) throws InterruptedException { - Set latestTableNames = tableExtension.list(conn, database.getName(), tableType) - .stream().map(DBObjectIdentity::getName).collect(Collectors.toCollection(LinkedHashSet::new)); + ConnectionConfig dataSource, List
tables, + Connection conn, DBObjectType tableType, Set latestTableNames) throws InterruptedException { if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { tables.addAll(latestTableNames.stream().map(tableName -> { Table table = new Table(); @@ -181,6 +201,8 @@ private DBSchemaSyncer getSyncerByTableType(@NotNull DBObjectType tableType) { return dbTableSyncer; case EXTERNAL_TABLE: return dbExternalTableSyncer; + case VIEW: + return dbViewSyncer; default: throw new IllegalArgumentException("Unsupported table type: " + tableType); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java index 7b0201b50a..be216bbfb2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.permission; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -277,7 +278,8 @@ public List filterUnauthorizedDBResources( } }); Set dbIds = resource2Types.keySet().stream().map(DBResource::getDatabaseId).collect(Collectors.toSet()); - List tbEntities = dbObjectRepository.findByDatabaseIdInAndType(dbIds, DBObjectType.TABLE); + List tbEntities = dbObjectRepository.findByDatabaseIdInAndTypeIn(dbIds, + Arrays.asList(DBObjectType.TABLE, DBObjectType.VIEW, DBObjectType.EXTERNAL_TABLE)); Map tbId2Entity = tbEntities.stream().collect(Collectors.toMap(DBObjectEntity::getId, e -> e)); Map> dbId2tbName2tbEntity = new HashMap<>(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java index a67b618fc3..fb1520c8f4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java @@ -116,7 +116,8 @@ public void process(CreateFlowInstanceReq req) { } table.setDatabaseId(database.getId()); table.setDatabaseName(database.getName()); - if (table.getType() == DBObjectType.TABLE) { + if (table.getType() == DBObjectType.TABLE || table.getType() == DBObjectType.VIEW + || table.getType() == DBObjectType.EXTERNAL_TABLE) { ConnectionConfig dataSource = id2DataSource.get(database.getDataSource().getId()); if (dataSource == null) { throw new NotFoundException(ResourceType.ODC_CONNECTION, "id", database.getDataSource().getId()); diff --git a/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestDBConfigurations.java b/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestDBConfigurations.java index b30b390ee5..f606dc7ad5 100644 --- a/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestDBConfigurations.java +++ b/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestDBConfigurations.java @@ -150,6 +150,11 @@ private void createTestDatabasesAndUpdateConfig(TestDBType type, TestDBConfigura sql = new StringBuilder("GRANT SYSDBA, RESOURCE, CREATE SESSION TO ").append(username); stmt.execute(sql.toString()); log.info("grant sysdba to new created user, username: {}", username); + // Although the above code has granted sysdba role to the new user, the connection created with the + // new user does not use the sysdba role, so the following grant statement is required + sql = new StringBuilder("GRANT ALL PRIVILEGES TO ").append(username); + stmt.execute(sql.toString()); + log.info("grant all privileges to new created user, username: {}", username); configuration.setDefaultDBName(username); configuration.setUsername(username); } From 8065d1043f28f221632512c46b9374cc7e734868 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Fri, 6 Dec 2024 15:02:50 +0800 Subject: [PATCH 048/118] feat(project): support delete projects (#3948) * support batch delete projects * list unfinished tickets * bugfix * fix archive project name * delete unstaged code * add unit tests * add TODO --- .../collaboration/ProjectServiceTest.java | 31 +++++++ .../odc/core/shared/constant/FlowStatus.java | 10 ++- .../web/controller/v2/ProjectController.java | 14 ++++ .../metadb/connection/DatabaseRepository.java | 2 +- .../iam/UserDatabasePermissionRepository.java | 5 ++ .../iam/UserTablePermissionRepository.java | 5 ++ .../metadb/schedule/ScheduleRepository.java | 2 +- .../collaboration/project/ProjectService.java | 81 +++++++++++++++++-- .../project/model/TicketReference.java | 35 ++++++++ .../odc/service/flow/FlowInstanceService.java | 15 +++- .../odc/service/schedule/ScheduleService.java | 12 +++ .../schedule/model/ScheduleStatus.java | 9 ++- 12 files changed, 211 insertions(+), 10 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/TicketReference.java diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java index 763a1c470a..899878ad9f 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java @@ -15,8 +15,14 @@ */ package com.oceanbase.odc.service.collaboration; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doNothing; + import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.UUID; @@ -36,6 +42,7 @@ import com.oceanbase.odc.core.shared.constant.ResourceRoleName; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.BadRequestException; +import com.oceanbase.odc.core.shared.exception.UnsupportedException; import com.oceanbase.odc.metadb.collaboration.ProjectEntity; import com.oceanbase.odc.metadb.collaboration.ProjectRepository; import com.oceanbase.odc.metadb.connection.DatabaseEntity; @@ -49,6 +56,7 @@ import com.oceanbase.odc.service.collaboration.project.model.Project.ProjectMember; import com.oceanbase.odc.service.collaboration.project.model.QueryProjectParams; import com.oceanbase.odc.service.collaboration.project.model.SetArchivedReq; +import com.oceanbase.odc.service.iam.ProjectPermissionValidator; import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.UserOrganizationService; import com.oceanbase.odc.service.iam.UserService; @@ -89,6 +97,9 @@ public class ProjectServiceTest extends ServiceTestEnv { @MockBean private ResourceRoleRepository resourceRoleRepository; + @MockBean + private ProjectPermissionValidator projectPermissionValidator; + @Before public void setUp() { Mockito.when(userService.nullSafeGet(Mockito.anyLong())).thenReturn(getUserEntity()); @@ -164,6 +175,7 @@ public void testUpdateProject_Success() { public void testArchiveProject_Archived() throws InterruptedException { Project saved = projectService.create(getProject()); Mockito.when(resourceRoleService.saveAll(Mockito.any())).thenReturn(listUserResourceRole(saved.getId())); + doNothing().when(projectPermissionValidator).checkProjectRole(anyCollection(), anyList()); SetArchivedReq req = new SetArchivedReq(); req.setArchived(true); Project archived = projectService.setArchived(saved.getId(), req); @@ -179,6 +191,25 @@ public void testArchiveProject_NotArchived() throws InterruptedException { projectService.setArchived(saved.getId(), req); } + @Test + public void testDeleteProject_ArchivedProject_Success() throws InterruptedException { + Project saved = projectService.create(getProject()); + Mockito.when(resourceRoleService.saveAll(Mockito.any())).thenReturn(listUserResourceRole(saved.getId())); + doNothing().when(projectPermissionValidator).checkProjectRole(anyCollection(), anyList()); + SetArchivedReq req = new SetArchivedReq(); + req.setArchived(true); + projectService.setArchived(saved.getId(), req); + Assert.assertTrue(projectService.batchDelete(new HashSet<>(Arrays.asList(saved.getId())))); + } + + @Test(expected = UnsupportedException.class) + public void testDeleteProject_NotArchivedProject_Fail() { + Project saved = projectService.create(getProject()); + Mockito.when(resourceRoleService.saveAll(Mockito.any())).thenReturn(listUserResourceRole(saved.getId())); + doNothing().when(projectPermissionValidator).checkProjectRole(anyCollection(), anyList()); + projectService.batchDelete(new HashSet<>(Arrays.asList(saved.getId()))); + } + @Test public void test_listBasicInfoForApply() { ProjectEntity entity = projectRepository.save(getProjectEntity()); diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java index 8469af1795..6504a046e0 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java @@ -15,6 +15,10 @@ */ package com.oceanbase.odc.core.shared.constant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + /** * @author wenniu.ly * @date 2022/2/9 @@ -87,6 +91,10 @@ public enum FlowStatus { */ COMPLETED, - PRE_CHECK_FAILED, + PRE_CHECK_FAILED; + public static List listUnfinishedStatus() { + return Collections.unmodifiableList( + Arrays.asList(CREATED, APPROVING, WAIT_FOR_EXECUTION, WAIT_FOR_CONFIRM, EXECUTING, ROLLBACKING)); + } } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ProjectController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ProjectController.java index b5e4178e43..f0b94657e5 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ProjectController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ProjectController.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.server.web.controller.v2; import java.util.List; +import java.util.Set; import javax.validation.Valid; @@ -35,6 +36,7 @@ import com.oceanbase.odc.service.collaboration.project.model.Project.ProjectMember; import com.oceanbase.odc.service.collaboration.project.model.QueryProjectParams; import com.oceanbase.odc.service.collaboration.project.model.SetArchivedReq; +import com.oceanbase.odc.service.collaboration.project.model.TicketReference; import com.oceanbase.odc.service.common.response.ListResponse; import com.oceanbase.odc.service.common.response.PaginatedResponse; import com.oceanbase.odc.service.common.response.Responses; @@ -131,4 +133,16 @@ public SuccessResponse setArchived(@PathVariable Long id, return Responses.success(projectService.setArchived(id, setArchivedReq)); } + @ApiOperation(value = "batchDeleteProject", notes = "Delete projects") + @RequestMapping(value = "/projects/batchDelete", method = RequestMethod.POST) + public SuccessResponse batchDelete(@RequestBody Set ids) throws InterruptedException { + return Responses.success(projectService.batchDelete(ids)); + } + + @ApiOperation(value = "listUnfinishedTickets", notes = "List unfinished ticket references of a project") + @RequestMapping(value = "/projects/{id:[\\d]+}/unfinishedTickets", method = RequestMethod.GET) + public SuccessResponse listUnfinishedTickets(@PathVariable Long id) { + return Responses.success(projectService.getProjectTicketReference(id)); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java index 1e1a27f678..3cb2f93412 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java @@ -42,7 +42,7 @@ public interface DatabaseRepository extends JpaRepository, List findByProjectId(Long projectId); - List findByProjectIdIn(List projectIds); + List findByProjectIdIn(Collection projectIds); List findByProjectIdAndExisted(Long projectId, Boolean existed); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionRepository.java index 7356b3c6ff..7fd21326a1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionRepository.java @@ -17,6 +17,7 @@ import java.util.Collection; import java.util.List; +import java.util.Set; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -43,4 +44,8 @@ List findNotExpiredByUserIdAndDatabaseIdIn(@Param( List findByUserIdAndProjectId(@Param("userId") Long userId, @Param("projectId") Long projectId); + @Query(value = "select v.* from list_user_database_permission_view v where v.project_id in (:projectIds)", + nativeQuery = true) + List findByProjectIdIn(@Param("projectIds") Set projectIds); + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java index f71ba5c8b2..da54af9546 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java @@ -17,6 +17,7 @@ import java.util.Collection; import java.util.List; +import java.util.Set; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -51,4 +52,8 @@ List findNotExpiredByUserIdAndTableIdIn(@Param("userI List findByUserIdAndProjectId(@Param("userId") Long userId, @Param("projectId") Long projectId); + @Query(value = "select v.* from list_user_table_permission_view v where v.project_id in (:projectIds)", + nativeQuery = true) + List findByProjectIdIn(@Param("projectIds") Set projectIds); + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java index affa258d22..469844b456 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java @@ -57,7 +57,7 @@ public interface ScheduleRepository extends OdcJpaRepository 0) { - throw new BadRequestException("Please disable all active tickets in the project first."); - } + checkUnfinishedTickets(id); previous.setArchived(true); + previous.setName(previous.getName() + "_archived_" + System.currentTimeMillis()); ProjectEntity saved = repository.save(previous); List connectionEntities = connectionConfigRepository.findByProjectId(id).stream() .peek(e -> e.setProjectId(null)).collect(Collectors.toList()); @@ -390,6 +397,41 @@ public boolean deleteProjectMember(@NonNull Long projectId, @NonNull Long userId return deleted; } + @Transactional(rollbackFor = Exception.class) + @SkipAuthorize("Internal usage") + public Boolean batchDelete(Set projectIds) { + projectPermissionValidator.checkProjectRole(projectIds, Collections.singletonList(ResourceRoleName.OWNER)); + repository.findByIdIn(projectIds).stream().forEach(project -> { + if (project.getBuiltin() || !project.getArchived()) { + throw new UnsupportedException(ErrorCodes.IllegalOperation, + new Object[] {"builtin or not-archived project, projectName=" + project.getName()}, + "Operation on builtin or not-archived project is not allowed"); + } + }); + repository.deleteAllById(projectIds); + // TODO: bind these resource delete logic in one place for better maintainability + resourceRoleService.deleteByResourceTypeAndIdIn(ResourceType.ODC_PROJECT, projectIds); + deleteMemberRelatedDatabasePermissions(projectIds); + deleteMemberRelatedTablePermissions(projectIds); + List relatedDatabaseIds = databaseRepository.findByProjectIdIn(projectIds).stream() + .map(DatabaseEntity::getId).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(relatedDatabaseIds)) { + resourceRoleService.deleteByResourceTypeAndIdIn(ResourceType.ODC_DATABASE, relatedDatabaseIds); + } + return true; + } + + @Transactional(rollbackFor = Exception.class) + @SkipAuthorize("Internal usage") + public TicketReference getProjectTicketReference(Long projectId) { + TicketReference reference = new TicketReference(); + reference.setUnfinishedFlowInstances( + flowInstanceService.listUnfinishedFlowInstances(Pageable.unpaged(), projectId).getContent()); + reference.setUnfinishedSchedules( + scheduleService.listUnfinishedSchedulesByProjectId(Pageable.unpaged(), projectId).getContent()); + return reference; + } + @SkipAuthorize @Transactional(rollbackFor = Exception.class) public boolean deleteProjectMemberSkipPermissionCheck(@NonNull Long projectId, @NonNull Long userId) { @@ -506,6 +548,17 @@ public void checkCurrentUserProjectRolePermissions(@NotNull Project project, projectPermissionValidator.checkProjectRole(project.getId(), roleNames); } + public void checkUnfinishedTickets(@NonNull Long projectId) { + if (scheduleService.getEnabledScheduleCountByProjectId(projectId) > 0) { + throw new BadRequestException( + "There exists unfinished schedule tasks in the project, please disable them before archiving the project."); + } + if (flowInstanceService.listUnfinishedFlowInstances(Pageable.unpaged(), projectId).hasContent()) { + throw new BadRequestException( + "There exists unfinished tickets in the project, please stop them before archiving the project."); + } + } + private Project entityToModel(ProjectEntity entity, List userResourceRoles) { Project project = entityToModelWithoutCurrentUser(entity, userResourceRoles); project.setCreator(currentInnerUser()); @@ -593,6 +646,24 @@ private void deleteMemberRelatedTablePermissions(@NonNull Long userId, @NonNull } } + private void deleteMemberRelatedDatabasePermissions(@NonNull Set projectIds) { + List permissionIds = userDatabasePermissionRepository.findByProjectIdIn(projectIds).stream() + .map(UserDatabasePermissionEntity::getId).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(permissionIds)) { + permissionRepository.deleteByIds(permissionIds); + userPermissionRepository.deleteByPermissionIds(permissionIds); + } + } + + private void deleteMemberRelatedTablePermissions(@NonNull Set projectIds) { + List permissionIds = userTablePermissionRepository.findByProjectIdIn(projectIds).stream() + .map(UserTablePermissionEntity::getId).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(permissionIds)) { + permissionRepository.deleteByIds(permissionIds); + userPermissionRepository.deleteByPermissionIds(permissionIds); + } + } + /** * 1. check if duplicate project name exists
* 2. check if all project members belong to current organization
diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/TicketReference.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/TicketReference.java new file mode 100644 index 0000000000..f2a48b5c5b --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/TicketReference.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.collaboration.project.model; + +import java.util.List; + +import com.oceanbase.odc.service.flow.model.FlowInstanceDetailResp; +import com.oceanbase.odc.service.schedule.model.ScheduleOverviewHist; + +import lombok.Data; + +/** + * @Author: Lebie + * @Date: 2024/11/28 17:24 + * @Description: [] + */ +@Data +public class TicketReference { + private List unfinishedFlowInstances; + + private List unfinishedSchedules; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java index 29c728fbc2..b8afd3fa7b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java @@ -414,7 +414,20 @@ public FlowMetaInfo getMetaInfo() { return FlowMetaInfo.of(entities); } - public Page listAll(@NotNull Pageable pageable, @NotNull QueryFlowInstanceParams params) { + public Page listUnfinishedFlowInstances(@NotNull Pageable pageable, + @NonNull Long projectId) { + QueryFlowInstanceParams.builder().projectIds(Collections.singleton(projectId)).containsAll(true).statuses( + Arrays.asList(FlowStatus.APPROVING, FlowStatus.CREATED, FlowStatus.EXECUTING, FlowStatus.ROLLBACKING, + FlowStatus.WAIT_FOR_EXECUTION, FlowStatus.WAIT_FOR_CONFIRM)) + .build(); + return list(pageable, + QueryFlowInstanceParams.builder().projectIds(Collections.singleton(projectId)).containsAll(true) + .statuses( + FlowStatus.listUnfinishedStatus()) + .build()); + } + + private Page listAll(@NotNull Pageable pageable, @NotNull QueryFlowInstanceParams params) { if (Objects.nonNull(params.getProjectIds())) { projectPermissionValidator.checkProjectRole(params.getProjectIds(), ResourceRoleName.all()); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 8e1d791602..4963daf12e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -44,6 +44,7 @@ import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cglib.beans.BeanMap; +import org.springframework.context.annotation.Lazy; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -166,6 +167,7 @@ public class ScheduleService { private ScheduleResponseMapperFactory scheduleResponseMapperFactory; @Autowired + @Lazy private ProjectService projectService; @Autowired @@ -794,6 +796,16 @@ public Page list(@NotNull Pageable pageable, @NotNull Quer : returnValue.map(o -> scheduleId2Overview.get(o.getId())); } + public Page listUnfinishedSchedulesByProjectId(@NonNull Pageable pageable, + @NonNull Long projectId) { + return list(pageable, QueryScheduleParams.builder().projectId(projectId) + .statuses(ScheduleStatus.listUnfinishedStatus()).build()); + } + + public int getEnabledScheduleCountByProjectId(@NonNull Long projectId) { + return scheduleRepository.getEnabledScheduleCountByProjectId(projectId); + } + public Page listScheduleOverview(@NotNull Pageable pageable, @NotNull QueryScheduleParams params) { log.info("List schedule overview req:{}", params); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java index 5f84136abf..ddad70d69b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java @@ -15,6 +15,10 @@ */ package com.oceanbase.odc.service.schedule.model; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + /** * @Author：tinker * @Date: 2022/11/16 15:36 @@ -36,6 +40,9 @@ public enum ScheduleStatus { COMPLETED, EXECUTION_FAILED, - DELETED + DELETED; + public static List listUnfinishedStatus() { + return Collections.unmodifiableList(Arrays.asList(CREATING, APPROVING, PAUSE, ENABLED)); + } } From 782d802ec96f72ceef873a9f02edf2b2b2e52a7d Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Thu, 12 Dec 2024 10:13:22 +0800 Subject: [PATCH 049/118] feat(session): SQL console connection keep alive (#3993) * support client keep alive * add ut * response to comments * do not reconstruct exectuor service when reset connection * make robust * optimize code * only keep alive in console connection --- .../SingleConnectionDataSource.java | 94 +++++++++++++++++-- .../SingleConnectionDataSourceTest.java | 50 ++++++++++ .../connection/database/DatabaseService.java | 5 +- .../oscfms/OscActionFsmBase.java | 2 +- .../factory/DefaultConnectSessionFactory.java | 11 ++- .../factory/OBConsoleDataSourceFactory.java | 12 ++- .../odc/service/sqlcheck/SqlCheckService.java | 2 +- .../task/base/precheck/PreCheckTask.java | 2 +- 8 files changed, 154 insertions(+), 24 deletions(-) diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSource.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSource.java index 59da5ee38d..a58943a422 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSource.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSource.java @@ -21,14 +21,19 @@ import java.lang.reflect.Proxy; import java.sql.Connection; import java.sql.SQLException; +import java.sql.Statement; import java.util.LinkedList; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import com.oceanbase.odc.common.concurrent.ExecutorUtils; import com.oceanbase.odc.common.event.AbstractEvent; import com.oceanbase.odc.common.event.EventPublisher; import com.oceanbase.odc.core.datasource.event.ConnectionResetEvent; @@ -66,15 +71,42 @@ public class SingleConnectionDataSource extends BaseClassBasedDataSource impleme protected volatile Connection connection; private final List initializerList = new LinkedList<>(); private volatile Lock lock; + private final boolean keepAlive; + private long keepAliveIntervalMillis = 5 * 60 * 1000; + @Getter + @Setter + private String keepAliveSql = "SELECT 1 FROM DUAL"; + @Getter + private final int maxFailedKeepAliveAttempts = 5; + private AtomicInteger failedKeepAliveAttempts = new AtomicInteger(0); + private ScheduledExecutorService keepAliveScheduler; @Setter private long timeOutMillis = 10 * 1000; public SingleConnectionDataSource() { - this(false); + this(false, false); + } + + public SingleConnectionDataSource(boolean autoReconnect, boolean keepAlive) { + this.autoReconnect = autoReconnect; + this.keepAlive = keepAlive; + initKeepAliveScheduler(); } - public SingleConnectionDataSource(boolean autoReconnect) { + public SingleConnectionDataSource(boolean autoReconnect, boolean keepAlive, long keepAliveIntervalMillis) { this.autoReconnect = autoReconnect; + this.keepAlive = keepAlive; + this.keepAliveIntervalMillis = keepAliveIntervalMillis; + initKeepAliveScheduler(); + } + + public SingleConnectionDataSource(boolean autoReconnect, boolean keepAlive, long keepAliveIntervalMillis, + String keepAliveSql) { + this.autoReconnect = autoReconnect; + this.keepAlive = keepAlive; + this.keepAliveIntervalMillis = keepAliveIntervalMillis; + this.keepAliveSql = keepAliveSql; + initKeepAliveScheduler(); } @Override @@ -122,9 +154,10 @@ public Connection getConnection(String username, String password) throws SQLExce */ public synchronized void resetConnection() throws SQLException { log.info("The connection will be reset soon"); - close(); + closeConnection(); this.connection = null; this.lock = null; + this.failedKeepAliveAttempts.set(0); try (Connection conn = innerCreateConnection()) { onConnectionReset(conn); } @@ -136,13 +169,8 @@ public void addInitializer(@NonNull ConnectionInitializer connectionInitializer) @Override public void close() { - if (!Objects.isNull(this.connection)) { - try { - this.connection.close(); - } catch (Throwable throwable) { - log.error("Failed to close the connection", throwable); - } - } + shutdownKeepAliveScheduler(); + closeConnection(); } protected void prepareConnection(Connection con) throws SQLException { @@ -152,6 +180,17 @@ protected void prepareConnection(Connection con) throws SQLException { } } + + private void closeConnection() { + if (!Objects.isNull(this.connection)) { + try { + this.connection.close(); + } catch (Throwable throwable) { + log.error("Failed to close the connection", throwable); + } + } + } + private boolean tryLock(Lock lock) { try { boolean locked = lock.tryLock(timeOutMillis, TimeUnit.MILLISECONDS); @@ -235,6 +274,41 @@ private synchronized Connection innerCreateConnection() throws SQLException { } } + + private void initKeepAliveScheduler() { + if (!keepAlive || Objects.nonNull(keepAliveScheduler)) { + return; + } + failedKeepAliveAttempts.set(0); + keepAliveScheduler = Executors.newScheduledThreadPool(1); + keepAliveScheduler.scheduleWithFixedDelay(() -> { + if (failedKeepAliveAttempts.get() > maxFailedKeepAliveAttempts) { + return; + } + try (Connection conn = getConnection()) { + try (Statement statement = conn.createStatement()) { + statement.execute(keepAliveSql); + failedKeepAliveAttempts.set(0); + log.debug("Keep connection alive success"); + } + } catch (Exception e) { + log.warn("Failed to keep connection alive", e); + failedKeepAliveAttempts.incrementAndGet(); + } + }, this.keepAliveIntervalMillis, this.keepAliveIntervalMillis, TimeUnit.MILLISECONDS); + } + + + private void shutdownKeepAliveScheduler() { + try { + if (Objects.nonNull(keepAliveScheduler) && !keepAliveScheduler.isTerminated()) { + ExecutorUtils.gracefulShutdown(keepAliveScheduler, "connection-keep-alive-executor", 5L); + } + } catch (Exception ex) { + log.warn("Failed to shutdown keep alive scheduler", ex); + } + } + /** * Invocation handler that suppresses close calls on JDBC Connections. * diff --git a/server/odc-core/src/test/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSourceTest.java b/server/odc-core/src/test/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSourceTest.java index 1fe5d3fb70..8e265bc1ee 100644 --- a/server/odc-core/src/test/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSourceTest.java +++ b/server/odc-core/src/test/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSourceTest.java @@ -15,6 +15,15 @@ */ package com.oceanbase.odc.core.datasource; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -140,6 +149,47 @@ public void testGetConnectionLock() throws Exception { Assert.assertTrue(exceptions.get(0) instanceof ConflictException); } + @Test + public void testKeepAlive_Enabled() throws SQLException, InterruptedException { + Connection mockConnection = mock(Connection.class); + Statement mockStatement = mock(Statement.class); + when(mockConnection.createStatement()).thenReturn(mockStatement); + doNothing().when(mockConnection).close(); + try (SingleConnectionDataSource spiedDataSource = spy(new SingleConnectionDataSource(false, true, 100) { + @Override + public Connection getConnection() { + return mockConnection; + } + })) { + doReturn(mockConnection).when(spiedDataSource).getConnection(); + + Thread.sleep(1000); + + verify(mockStatement, atLeastOnce()).execute(spiedDataSource.getKeepAliveSql()); + } + } + + @Test + public void testKeepAlive_Disabled() throws SQLException, InterruptedException { + Connection mockConnection = mock(Connection.class); + Statement mockStatement = mock(Statement.class); + when(mockConnection.createStatement()).thenReturn(mockStatement); + doNothing().when(mockConnection).close(); + try (SingleConnectionDataSource spiedDataSource = spy(new SingleConnectionDataSource(false, false, 100) { + @Override + public Connection getConnection() { + return mockConnection; + } + })) { + doReturn(mockConnection).when(spiedDataSource).getConnection(); + + Thread.sleep(1000); + + verify(mockStatement, never()).execute(spiedDataSource.getKeepAliveSql()); + } + + } + private void checkConnection(SingleConnectionDataSource dataSource) throws SQLException { try (Connection connection = dataSource.getConnection()) { try (Statement statement = connection.createStatement()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index 38410c4c26..26abc36750 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -364,7 +364,7 @@ public Database create(@NonNull CreateDatabaseReq req) { || !connectionService.checkPermission(req.getDataSourceId(), Collections.singletonList("update"))) { throw new AccessDeniedException(); } - DataSource dataSource = new OBConsoleDataSourceFactory(connection, true, false).getDataSource(); + DataSource dataSource = new OBConsoleDataSourceFactory(connection, true, false, false).getDataSource(); try (Connection conn = dataSource.getConnection()) { createDatabase(req, conn, connection); DBDatabase dbDatabase = dbSchemaService.detail(connection.getDialectType(), conn, req.getName()); @@ -658,7 +658,8 @@ private void syncTeamDataSources(ConnectionConfig connection) } private OBConsoleDataSourceFactory getDataSourceFactory(ConnectionConfig connection) { - OBConsoleDataSourceFactory obConsoleDataSourceFactory = new OBConsoleDataSourceFactory(connection, true, false); + OBConsoleDataSourceFactory obConsoleDataSourceFactory = + new OBConsoleDataSourceFactory(connection, true, false, false); LocalEventPublisher localEventPublisher = new LocalEventPublisher(); localEventPublisher.addEventListener(new GetConnectionFailedEventListener()); obConsoleDataSourceFactory.setEventPublisher(localEventPublisher); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java index 417e4c302d..6ba26d5121 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java @@ -411,7 +411,7 @@ public ConnectionConfig connectionConfig() { public ConnectionSession createConnectionSession() { ConnectionConfig connectionConfig = connectionConfig(); ConnectionSession connectionSession = - new DefaultConnectSessionFactory(connectionConfig, null, null, false).generateSession(); + new DefaultConnectSessionFactory(connectionConfig, null, null, false, false).generateSession(); ConnectionSessionUtil.setCurrentSchema(connectionSession, dbName); return connectionSession; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java index 3d14b881c0..c134756210 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java @@ -71,13 +71,15 @@ public class DefaultConnectSessionFactory implements ConnectionSessionFactory { private final Boolean autoCommit; private final EventPublisher eventPublisher; private final boolean autoReconnect; + private final boolean keepAlive; @Setter private long sessionTimeoutMillis; @Setter private ConnectionSessionIdGenerator idGenerator; public DefaultConnectSessionFactory(@NonNull ConnectionConfig connectionConfig, - Boolean autoCommit, TaskManagerFactory taskManagerFactory, boolean autoReconnect) { + Boolean autoCommit, TaskManagerFactory taskManagerFactory, boolean autoReconnect, + boolean keepAlive) { this.sessionTimeoutMillis = TimeUnit.MILLISECONDS.convert( ConnectionSessionConstants.SESSION_EXPIRATION_TIME_SECONDS, TimeUnit.SECONDS); this.connectionConfig = connectionConfig; @@ -86,15 +88,16 @@ public DefaultConnectSessionFactory(@NonNull ConnectionConfig connectionConfig, this.eventPublisher = new LocalEventPublisher(); this.idGenerator = new DefaultConnectSessionIdGenerator(); this.autoReconnect = autoReconnect; + this.keepAlive = keepAlive; } public DefaultConnectSessionFactory(@NonNull ConnectionConfig connectionConfig, Boolean autoCommit, TaskManagerFactory taskManagerFactory) { - this(connectionConfig, autoCommit, taskManagerFactory, true); + this(connectionConfig, autoCommit, taskManagerFactory, true, true); } public DefaultConnectSessionFactory(@NonNull ConnectionConfig connectionConfig) { - this(connectionConfig, null, null, true); + this(connectionConfig, null, null, true, false); } @Override @@ -109,7 +112,7 @@ public ConnectionSession generateSession() { private void registerConsoleDataSource(ConnectionSession session) { OBConsoleDataSourceFactory dataSourceFactory = - new OBConsoleDataSourceFactory(connectionConfig, autoCommit, true, autoReconnect); + new OBConsoleDataSourceFactory(connectionConfig, autoCommit, true, autoReconnect, keepAlive); try { JdbcUrlParser urlParser = ConnectionPluginUtil .getConnectionExtension(connectionConfig.getDialectType()) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/OBConsoleDataSourceFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/OBConsoleDataSourceFactory.java index 9f68f7ae2c..7fa3656a8d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/OBConsoleDataSourceFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/OBConsoleDataSourceFactory.java @@ -78,6 +78,7 @@ public class OBConsoleDataSourceFactory implements CloneableDataSourceFactory { protected UserRole userRole; private String catalogName; private boolean autoReConnect; + private boolean keepAlive; private Map parameters; protected final ConnectionConfig connectionConfig; private final Boolean autoCommit; @@ -87,16 +88,16 @@ public class OBConsoleDataSourceFactory implements CloneableDataSourceFactory { protected final ConnectionExtensionPoint connectionExtensionPoint; public OBConsoleDataSourceFactory(@NonNull ConnectionConfig connectionConfig, Boolean autoCommit) { - this(connectionConfig, autoCommit, true); + this(connectionConfig, autoCommit, true, true, false); } public OBConsoleDataSourceFactory(@NonNull ConnectionConfig connectionConfig, - Boolean autoCommit, boolean initConnection) { - this(connectionConfig, autoCommit, initConnection, true); + Boolean autoCommit, boolean initConnection, boolean keepAlive) { + this(connectionConfig, autoCommit, initConnection, true, keepAlive); } public OBConsoleDataSourceFactory(@NonNull ConnectionConfig connectionConfig, - Boolean autoCommit, boolean initConnection, boolean autoReConnect) { + Boolean autoCommit, boolean initConnection, boolean autoReConnect, boolean keepAlive) { this.autoCommit = autoCommit; this.connectionConfig = connectionConfig; this.initConnection = initConnection; @@ -111,6 +112,7 @@ public OBConsoleDataSourceFactory(@NonNull ConnectionConfig connectionConfig, this.catalogName = connectionConfig.getCatalogName(); this.parameters = getJdbcParams(connectionConfig); this.autoReConnect = autoReConnect; + this.keepAlive = keepAlive; this.connectionExtensionPoint = ConnectionPluginUtil.getConnectionExtension(connectionConfig.getDialectType()); } @@ -202,7 +204,7 @@ public static Map getJdbcParams(@NonNull ConnectionConfig connec @Override public DataSource getDataSource() { String jdbcUrl = getJdbcUrl(); - SingleConnectionDataSource dataSource = new SingleConnectionDataSource(autoReConnect); + SingleConnectionDataSource dataSource = new SingleConnectionDataSource(autoReConnect, keepAlive); dataSource.setEventPublisher(eventPublisher); dataSource.setUrl(jdbcUrl); dataSource.setUsername(username); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/SqlCheckService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/SqlCheckService.java index 7b6359ba7f..bf7be02bde 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/SqlCheckService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/SqlCheckService.java @@ -141,7 +141,7 @@ public List check(@NotNull Long environmentId, @NonNull String d } Environment env = this.environmentService.detail(environmentId); List rules = this.ruleService.list(env.getRulesetId(), QueryRuleMetadataParams.builder().build()); - OBConsoleDataSourceFactory factory = new OBConsoleDataSourceFactory(config, true, false); + OBConsoleDataSourceFactory factory = new OBConsoleDataSourceFactory(config, true, false, false); factory.resetSchema(origin -> OBConsoleDataSourceFactory.getSchema(databaseName, config.getDialectType())); SqlCheckContext checkContext = new SqlCheckContext((long) sqls.size()); try (SingleConnectionDataSource dataSource = (SingleConnectionDataSource) factory.getDataSource()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java index d51ffb633b..9f5a2ea994 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java @@ -244,7 +244,7 @@ private List checkViolations(List sqls) { } ConnectionConfig config = this.parameters.getConnectionConfig(); List rules = this.parameters.getRules(); - OBConsoleDataSourceFactory factory = new OBConsoleDataSourceFactory(config, true, false); + OBConsoleDataSourceFactory factory = new OBConsoleDataSourceFactory(config, true, false, false); factory.resetSchema(origin -> OBConsoleDataSourceFactory .getSchema(this.parameters.getRiskLevelDescriber().getDatabaseName(), config.getDialectType())); SqlCheckContext checkContext = new SqlCheckContext((long) sqls.size()); From ad9876163badfe8233b42bd81de6cc2254087fac Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Fri, 13 Dec 2024 15:34:45 +0800 Subject: [PATCH 050/118] feat(pl debug): pl debugging adapt odp directional routing (#3938) * finish pl debugging adapt odp directional routing * Add ODP version check for CLOUD_OB_ORACLE in DBPLOperators * Add ODP version check for CLOUD_OB_ORACLE in DBPLOperators * Add PLDebugODPSpecifiedRoute to GetPLErrorCallBack and DebuggerSession * Fix log message and SQL syntax in OraclePLOperator and PLUtils * Optimize PLDebugODPSpecifiedRoute usage in OBOracleCallFunctionCallBack * Refactor OBProxy version check and routing in DBPLOperators * modify code format * Refactor OBProxy version handling and add ODP version support * Enhance ODP routing logic in AbstractDebugSession * modify code format * Refactor CallProcedureCallBack to handle null PLDebugODPSpecifiedRoute * Refactor OBOracleCallFunctionCallBack to handle null route * Refactor ODP routing logic and remove unused imports * modify code format * Enhance ODP routing with PLUtils in PLDebugSession * Add `@NonNull` annotations to PLDebugODPSpecifiedRoute constructor * Refactor ODP routing logic and enhance error handling in AbstractDebugSession * Remove unused getODPVersion methods and update ODP routing logic * Remove trailing whitespace in Postgres and Oracle extensions --- .../session/ConnectionSessionConstants.java | 5 + .../core/session/ConnectionSessionUtil.java | 22 ++++ .../oceanbase/odc/core/sql/util/OBUtils.java | 14 +++ .../connection/util/ConnectionInfoUtil.java | 13 +++ .../db/session/DefaultDBSessionManage.java | 26 +---- .../model/PLDebugODPSpecifiedRoute.java | 38 +++++++ .../pldebug/operator/DBPLOperators.java | 12 ++ .../pldebug/operator/GetPLErrorCallBack.java | 10 +- .../pldebug/operator/OraclePLOperator.java | 2 +- .../pldebug/session/AbstractDebugSession.java | 104 ++++++++++++++---- .../pldebug/session/DebuggeeSession.java | 6 +- .../pldebug/session/DebuggerSession.java | 20 ++-- .../pldebug/session/PLDebugSession.java | 10 +- .../pldebug/util/CallProcedureCallBack.java | 15 +++ .../util/OBOracleCallFunctionCallBack.java | 27 ++++- .../odc/service/pldebug/util/PLDebugTask.java | 3 +- .../odc/service/pldebug/util/PLUtils.java | 16 +++ .../factory/DefaultConnectSessionFactory.java | 1 + 18 files changed, 281 insertions(+), 63 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/model/PLDebugODPSpecifiedRoute.java diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionConstants.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionConstants.java index aae3896b6e..639f144bc2 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionConstants.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionConstants.java @@ -74,6 +74,11 @@ public class ConnectionSessionConstants { * form of attributes, this is the key */ public static final String OB_VERSION = "OB_VERSION"; + /** + * The odp version needs to be stored in the database session in the form of attributes, this is the + * key + */ + public static final String ODP_VERSION = "ODP_VERSION"; /** * The connection account type current database session needs to be stored in the database session * in the form of attributes, this is the key diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionUtil.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionUtil.java index 9aa1ccbc4c..6d3136d8af 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionUtil.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionUtil.java @@ -44,6 +44,7 @@ import com.oceanbase.odc.common.lang.Pair; import com.oceanbase.odc.common.util.HashUtils; import com.oceanbase.odc.common.util.SystemUtils; +import com.oceanbase.odc.common.util.VersionUtils; import com.oceanbase.odc.core.datasource.CloneableDataSourceFactory; import com.oceanbase.odc.core.shared.Verify; import com.oceanbase.odc.core.shared.constant.DialectType; @@ -72,6 +73,8 @@ @Slf4j public class ConnectionSessionUtil { + private static final String ODP_SPECIFIED_ROUTINE_ENABLED_VERSION_NUMBER = "3.1.11"; + public static void logSocketInfo(Connection connection, String scenario) { if (!(connection instanceof OceanBaseConnection)) { log.debug("skip log connection socket info due not an OceanBaseConnection, className={}, scenario={}", @@ -616,6 +619,25 @@ public static String getUniqueIdentifier(@NonNull ConnectionSession connectionSe return HashUtils.md5(connectionSession.getId()).replace("-", ""); } + /** + * Get the OBProxy version number. If an exception occurs or the version does not support, return + * null. + * + * @param connectionSession + * @return + */ + public static String getObProxyVersion(@NonNull ConnectionSession connectionSession) { + return (String) connectionSession.getAttribute(ConnectionSessionConstants.ODP_VERSION); + } + + public static boolean isSupportObProxyRoute(String obProxyVersion) { + if (obProxyVersion == null + || VersionUtils.isLessThan(obProxyVersion, ODP_SPECIFIED_ROUTINE_ENABLED_VERSION_NUMBER)) { + return false; + } + return true; + } + private static String getOrCreateFullPathAppendingSuffixToDataPath(@NonNull String suffix) throws IOException { String dataDir = SystemUtils.getEnvOrProperty("file.storage.dir"); if (dataDir == null) { diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java index 78bcfd7d59..433f11aea0 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java @@ -308,6 +308,20 @@ public static String getObVersion(Connection connection) { } } + public static String getODPVersion(Connection connection) { + try (Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery("select proxy_version()")) { + if (resultSet.next()) { + return resultSet.getString("proxy_version()"); + } else { + return null; + } + } + } catch (Exception e) { + return null; + } + } + private static String buildSqlAudit(ConnectType connectType, String version) { StringBuilder str = new StringBuilder(); if (connectType.getDialectType().isOracle()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/util/ConnectionInfoUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/util/ConnectionInfoUtil.java index 60a56cdaa6..0fce0f67c6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/util/ConnectionInfoUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/util/ConnectionInfoUtil.java @@ -93,6 +93,19 @@ public static void initSessionVersion(@NonNull ConnectionSession connectionSessi log.debug("Init DB version completed."); } + public static void initOdpVersionIfExists(@NonNull ConnectionSession connectionSession) { + DialectType dialectType = connectionSession.getDialectType(); + if (dialectType != null && dialectType.isOceanbase()) { + String odpVersion = getSyncJdbcExecutor(connectionSession).execute(OBUtils::getODPVersion); + if (odpVersion == null) { + log.debug("OB Proxy does not exist or failed to obtain OB Proxy version."); + return; + } + connectionSession.setAttribute(ConnectionSessionConstants.ODP_VERSION, odpVersion); + log.debug("Init OB Proxy version completed."); + } + } + public static void killQuery(@NonNull String connectionId, @NonNull DataSourceFactory dataSourceFactory, DialectType dialectType) throws Exception { DataSource dataSource = dataSourceFactory.getDataSource(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index cfebed9a27..6f92c763a0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -238,7 +238,10 @@ private List additionalKillIfNecessary(ConnectionSession conn Collectors.toMap(s -> s.getSqlTuple().getSqlId(), SqlTupleSessionId::getSessionId)); Boolean isDirectedOBServer = isObServerDirected(connectionSession); - String obProxyVersion = getObProxyVersion(connectionSession, isDirectedOBServer); + String obProxyVersion = null; + if (Boolean.FALSE.equals(isDirectedOBServer)) { + obProxyVersion = ConnectionSessionUtil.getObProxyVersion(connectionSession); + } String obVersion = ConnectionSessionUtil.getVersion(connectionSession); boolean isEnabledGlobalClientSession = isGlobalClientSessionEnabled(connectionSession, obProxyVersion, obVersion); @@ -288,27 +291,6 @@ private Boolean isObServerDirected(ConnectionSession connectionSession) { return null; } - /** - * Get the OBProxy version number. If an exception occurs or the version does not support, return - * null. - * - * @param connectionSession - * @param isDirectedOBServer - * @return - */ - private String getObProxyVersion(ConnectionSession connectionSession, Boolean isDirectedOBServer) { - if (Boolean.TRUE.equals(isDirectedOBServer)) { - return null; - } - try { - return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) - .queryForObject("select proxy_version()", String.class); - } catch (Exception e) { - log.warn("Failed to obtain the OBProxy version number: {}", e.getMessage()); - return null; - } - } - private boolean isGlobalClientSessionEnabled(ConnectionSession connectionSession, String obProxyVersion, String obVersion) { // verification version requirement diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/model/PLDebugODPSpecifiedRoute.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/model/PLDebugODPSpecifiedRoute.java new file mode 100644 index 0000000000..57449ed384 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/model/PLDebugODPSpecifiedRoute.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.pldebug.model; + +import lombok.Getter; +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/12/2 15:20 + * @since: 4.3.3 + */ +@Getter +public class PLDebugODPSpecifiedRoute { + + private final String observerHost; + + private final Integer observerPort; + + public PLDebugODPSpecifiedRoute(@NonNull String observerHost, @NonNull Integer observerPort) { + this.observerHost = observerHost; + this.observerPort = observerPort; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/DBPLOperators.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/DBPLOperators.java index c3a48d3d3e..522b21f135 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/DBPLOperators.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/DBPLOperators.java @@ -21,6 +21,9 @@ import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.exception.UnsupportedException; +import lombok.extern.slf4j.Slf4j; + +@Slf4j public class DBPLOperators { public static DBPLOperator create(ConnectionSession connectionSession) { @@ -33,8 +36,17 @@ public static DBPLOperator create(ConnectionSession connectionSession) { if (connectType == ConnectType.OB_ORACLE) { return new OraclePLOperator(connectionSession); + } else if (connectType == ConnectType.CLOUD_OB_ORACLE) { + String obProxyVersion = ConnectionSessionUtil.getObProxyVersion(connectionSession); + if (ConnectionSessionUtil.isSupportObProxyRoute(obProxyVersion)) { + return new OraclePLOperator(connectionSession); + } else { + throw new UnsupportedException(String.format( + "ODP version not supported, the version number must be greater than or equal to 3.1.11")); + } } else { throw new UnsupportedException(String.format("ConnectType '%s' not supported", connectType)); } } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/GetPLErrorCallBack.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/GetPLErrorCallBack.java index 7c8822f97d..0f8cf95aa0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/GetPLErrorCallBack.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/GetPLErrorCallBack.java @@ -32,6 +32,7 @@ import com.oceanbase.odc.core.shared.exception.OBException; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.pldebug.model.DBPLError; +import com.oceanbase.odc.service.pldebug.model.PLDebugODPSpecifiedRoute; import com.oceanbase.odc.service.pldebug.util.PLUtils; /** @@ -42,10 +43,13 @@ public class GetPLErrorCallBack implements ConnectionCallback> { private final ConnectionConfig connectionConfig; private final DBPLError dbplError; + private final PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute; - public GetPLErrorCallBack(ConnectionConfig connectionConfig, DBPLError dbplError) { + public GetPLErrorCallBack(ConnectionConfig connectionConfig, DBPLError dbplError, + PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute) { this.connectionConfig = connectionConfig; this.dbplError = dbplError; + this.plDebugODPSpecifiedRoute = plDebugODPSpecifiedRoute; } @Override @@ -60,8 +64,10 @@ public List doInConnection(@Nullable Connection connection) throws Da tableName = "dba_errors"; } String schema = connectionConfig.getDefaultSchema(); + String specifiedRoute = PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute); String sql = String.format( - "select * from %s WHERE type = '%s' and name = '%s' and owner = '%s' order by line asc;", + "%s select * from %s WHERE type = '%s' and name = '%s' and owner = '%s' order by line asc;", + specifiedRoute, tableName, dbplError.getType(), dbplError.getName(), schema); List dbplErrors = new ArrayList<>(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/OraclePLOperator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/OraclePLOperator.java index 236ea933d6..b4050c3bfb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/OraclePLOperator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/OraclePLOperator.java @@ -75,7 +75,7 @@ public Boolean isSupportPLDebug() { try { syncJdbcExecutor.execute("CALL DBMS_OUTPUT.NEW_LINE()"); } catch (DataAccessException dataAccessException) { - log.warn("CALL DBMS_DEBUG.PING() occur error {}", dataAccessException.getMessage()); + log.warn("CALL DBMS_OUTPUT.NEW_LINE() occur error {}", dataAccessException.getMessage()); String result = dataAccessException.getMessage(); if (result != null && result.contains("Unknown")) { return false; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/AbstractDebugSession.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/AbstractDebugSession.java index 124d08576c..4e707b24bf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/AbstractDebugSession.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/AbstractDebugSession.java @@ -20,11 +20,11 @@ import java.sql.Statement; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.function.Supplier; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.springframework.jdbc.core.ConnectionCallback; import org.springframework.jdbc.core.JdbcOperations; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.SingleConnectionDataSource; @@ -34,15 +34,20 @@ import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionConstants; import com.oceanbase.odc.core.session.ConnectionSessionUtil; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.OdcConstants; import com.oceanbase.odc.core.shared.exception.OBException; import com.oceanbase.odc.core.shared.exception.UnexpectedException; import com.oceanbase.odc.core.shared.model.OdcDBSession; import com.oceanbase.odc.core.sql.util.OdcDBSessionRowMapper; +import com.oceanbase.odc.plugin.connect.api.HostAddress; import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.pldebug.model.PLDebugODPSpecifiedRoute; import com.oceanbase.odc.service.pldebug.util.CallProcedureCallBack; import com.oceanbase.odc.service.pldebug.util.OBOracleCallFunctionCallBack; +import com.oceanbase.odc.service.pldebug.util.PLUtils; import com.oceanbase.odc.service.session.initializer.BackupInstanceInitializer; import com.oceanbase.odc.service.session.initializer.DataSourceInitScriptInitializer; import com.oceanbase.tools.dbbrowser.model.DBFunction; @@ -71,6 +76,7 @@ public abstract class AbstractDebugSession implements AutoCloseable { protected DebugDataSource newDataSource; protected JdbcOperations jdbcOperations; protected DialectType dialectType; + protected PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute; private static final String OB_JDBC_PROTOCOL = "oceanbase"; public abstract boolean detectSessionAlive(); @@ -79,8 +85,15 @@ public List executeProcedure(DBProcedure procedure) { try { // -1 means statement queryTimeout will be default 0, // By default there is no limit on the amount of time allowed for a running statement to complete - return getJdbcOperations().execute( - new CallProcedureCallBack(procedure, -1, getSqlBuilder())); + CallProcedureCallBack callProcedureCallBack; + if (this.plDebugODPSpecifiedRoute == null) { + callProcedureCallBack = + new CallProcedureCallBack(procedure, -1, getSqlBuilder()); + } else { + callProcedureCallBack = + new CallProcedureCallBack(procedure, -1, getSqlBuilder(), this.plDebugODPSpecifiedRoute); + } + return getJdbcOperations().execute(callProcedureCallBack); } catch (Exception e) { throw OBException.executePlFailed(String.format("Error occurs when calling procedure={%s}, message=%s", procedure.getProName(), e.getMessage())); @@ -90,10 +103,16 @@ public List executeProcedure(DBProcedure procedure) { public DBFunction executeFunction(DBFunction dbFunction) { // -1 means statement queryTimeout will be default 0, // By default there is no limit on the amount of time allowed for a running statement to complete - ConnectionCallback functionConnectionCallback = - new OBOracleCallFunctionCallBack(dbFunction, -1); + OBOracleCallFunctionCallBack obOracleCallFunctionCallBack; + if (this.plDebugODPSpecifiedRoute == null) { + obOracleCallFunctionCallBack = + new OBOracleCallFunctionCallBack(dbFunction, -1); + } else { + obOracleCallFunctionCallBack = + new OBOracleCallFunctionCallBack(dbFunction, -1, this.plDebugODPSpecifiedRoute); + } try { - return getJdbcOperations().execute(functionConnectionCallback); + return getJdbcOperations().execute(obOracleCallFunctionCallBack); } catch (Exception e) { throw OBException.executePlFailed(String.format("Error occurs when calling dbFunction={%s}, message=%s", dbFunction.getFunName(), e.getMessage())); @@ -111,21 +130,57 @@ protected void acquireNewConnection(ConnectionSession connectionSession, this.connection = newDataSource.getConnection(); } - protected DebugDataSource acquireDataSource(ConnectionSession connectionSession, List initSqls) { + protected DebugDataSource acquireDataSource(@NonNull ConnectionSession connectionSession, + @NonNull List initSqls) { ConnectionConfig config = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(connectionSession); - DebugDataSource dataSource = new DebugDataSource(config, initSqls); String schema = ConnectionSessionUtil.getCurrentSchema(connectionSession); - String host; - Integer port; - if (StringUtils.isBlank(config.getClusterName())) { - host = config.getHost(); - port = config.getPort(); + Verify.equals(DialectType.OB_ORACLE, config.getDialectType(), "Only support OB_ORACLE"); + if (connectionSession.getConnectType() == ConnectType.OB_ORACLE + && StringUtils.isBlank(config.getClusterName())) { + // current connection is a direct observer + return buildDataSource(config, initSqls, null, + buildJdbcUrl(new HostAddress(config.getHost(), config.getPort()), schema)); } else { - String directServerIp = getDirectServerIp(connectionSession); - host = directServerIp.split(":")[0]; - port = Integer.parseInt(directServerIp.split(":")[1]); + Optional odpSpecifiedRouteDataSource = + tryGetODPSpecifiedRouteDataSource(connectionSession, initSqls, config, + schema); + if (odpSpecifiedRouteDataSource.isPresent()) { + return odpSpecifiedRouteDataSource.get(); + } + // cloud ob oracle not support direct connection to observer + if (connectionSession.getConnectType() == ConnectType.CLOUD_OB_ORACLE) { + throw new IllegalStateException(String.format( + "ODP specified route is not supported for cloud ob oracle connection, ODP version: %s", + ConnectionSessionUtil.getObProxyVersion(connectionSession))); + } + // use direct connection observer + return buildDataSource(config, initSqls, null, + buildJdbcUrl(getOBServerHostAddress(connectionSession), schema)); + } + } + + private Optional tryGetODPSpecifiedRouteDataSource(ConnectionSession connectionSession, + List initSqls, + ConnectionConfig config, String schema) { + String obProxyVersion = ConnectionSessionUtil.getObProxyVersion(connectionSession); + if (ConnectionSessionUtil.isSupportObProxyRoute(obProxyVersion)) { + HostAddress directServerIp = getOBServerHostAddress(connectionSession); + this.plDebugODPSpecifiedRoute = + new PLDebugODPSpecifiedRoute(directServerIp.getHost(), directServerIp.getPort()); + return Optional.of(buildDataSource(config, initSqls, this.plDebugODPSpecifiedRoute, + buildJdbcUrl(new HostAddress(config.getHost(), config.getPort()), schema))); } - String url = String.format("jdbc:%s://%s:%d/\"%s\"", OB_JDBC_PROTOCOL, host, port, schema); + return Optional.empty(); + } + + private String buildJdbcUrl(HostAddress hostAddress, String schema) { + return String.format("jdbc:%s://%s:%d/\"%s\"", OB_JDBC_PROTOCOL, hostAddress.getHost(), hostAddress.getPort(), + schema); + } + + private DebugDataSource buildDataSource(ConnectionConfig config, List initSqls, + PLDebugODPSpecifiedRoute route, String url) { + DebugDataSource dataSource = new DebugDataSource(config, initSqls, route); dataSource.setUrl(url); dataSource.setUsername(buildUserName(config)); dataSource.setPassword(config.getPassword()); @@ -133,7 +188,7 @@ protected DebugDataSource acquireDataSource(ConnectionSession connectionSession, return dataSource; } - private String getDirectServerIp(ConnectionSession connectionSession) { + private HostAddress getOBServerHostAddress(ConnectionSession connectionSession) { List sessions = connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY) .query("show full processlist", new OdcDBSessionRowMapper()); @@ -150,7 +205,8 @@ private String getDirectServerIp(ConnectionSession connectionSession) { if (StringUtils.isEmpty(directServerIp)) { throw new UnexpectedException("Empty direct server ip and port from 'show full processlist'"); } - return directServerIp; + String[] ipParts = directServerIp.split(":"); + return new HostAddress(ipParts[0], Integer.parseInt(ipParts[1])); } private String buildUserName(ConnectionConfig connectionConfig) { @@ -182,7 +238,8 @@ public void close() { protected void enableDbmsOutput(Statement statement) { try { - statement.execute(String.format("call dbms_output.enable(%s);", PL_LOG_CACHE_SIZE)); + statement.execute(String.format("%s call dbms_output.enable(%s);", + PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute), PL_LOG_CACHE_SIZE)); } catch (Exception e) { log.warn("enable dbms output failed, dbms_output may not exists, sid={}, reason={}", connectionSession.getId(), @@ -194,11 +251,14 @@ static class DebugDataSource extends SingleConnectionDataSource { private final List initSqls; private final List initializers; + private final PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute; - public DebugDataSource(@NonNull ConnectionConfig connectionConfig, List initSqls) { + public DebugDataSource(@NonNull ConnectionConfig connectionConfig, List initSqls, + PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute) { this.initSqls = initSqls; this.initializers = Arrays.asList(new BackupInstanceInitializer(connectionConfig), new DataSourceInitScriptInitializer(connectionConfig, true)); + this.plDebugODPSpecifiedRoute = plDebugODPSpecifiedRoute; } @Override @@ -207,7 +267,7 @@ protected void prepareConnection(Connection con) throws SQLException { if (CollectionUtils.isNotEmpty(this.initSqls)) { try (Statement statement = con.createStatement()) { for (String stmt : this.initSqls) { - statement.execute(stmt); + statement.execute(PLUtils.getSpecifiedRoute(plDebugODPSpecifiedRoute) + stmt); } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggeeSession.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggeeSession.java index 7405771b94..f633892042 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggeeSession.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggeeSession.java @@ -36,6 +36,7 @@ import com.oceanbase.odc.service.pldebug.model.PLDebugResult; import com.oceanbase.odc.service.pldebug.model.StartPLDebugReq; import com.oceanbase.odc.service.pldebug.util.PLDebugTask; +import com.oceanbase.odc.service.pldebug.util.PLUtils; import com.oceanbase.tools.dbbrowser.model.DBPLParam; import com.oceanbase.tools.dbbrowser.model.DBPLParamMode; import com.oceanbase.tools.dbbrowser.model.DBProcedure; @@ -77,7 +78,8 @@ public DebuggeeSession(ConnectionSession connectionSession, ThreadPoolExecutor d DBProcedure.of("DBMS_DEBUG", "SET_TIMEOUT_BEHAVIOUR", Collections.singletonList(param)); executeProcedure(dbProcedure); // 初始化debug_id - stmt.executeQuery("select dbms_debug.initialize() from dual;"); + stmt.executeQuery(PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute) + + "select dbms_debug.initialize() from dual;"); try (ResultSet resultSet = stmt.getResultSet()) { if (resultSet.next()) { debugId = resultSet.getString(1); @@ -86,7 +88,7 @@ public DebuggeeSession(ConnectionSession connectionSession, ThreadPoolExecutor d // 打开pl的日志输出 enableDbmsOutput(stmt); // 打开调试开关 - stmt.execute("call dbms_debug.debug_on();"); + stmt.execute(PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute) + "call dbms_debug.debug_on();"); } catch (SQLSyntaxErrorException e) { if (Objects.equals(e.getErrorCode(), ERROR_CODE) && StringUtils.contains(e.getMessage(), PL_DEBUGGING_ERROR_CODE)) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggerSession.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggerSession.java index 2fe1c90edf..925482bc22 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggerSession.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggerSession.java @@ -56,6 +56,7 @@ import com.oceanbase.odc.service.pldebug.model.StartPLDebugReq; import com.oceanbase.odc.service.pldebug.operator.DBPLOperators; import com.oceanbase.odc.service.pldebug.operator.GetPLErrorCallBack; +import com.oceanbase.odc.service.pldebug.util.PLUtils; import com.oceanbase.tools.dbbrowser.model.DBBasicPLObject; import com.oceanbase.tools.dbbrowser.model.DBFunction; import com.oceanbase.tools.dbbrowser.model.DBObjectType; @@ -107,16 +108,19 @@ public DebuggerSession(DebuggeeSession debuggeeSession, StartPLDebugReq req, boo debugType = req.getDebugType(); ddl = req.getAnonymousBlock(); this.syncEnabled = syncEnabled; - + if (debuggeeSession.getPlDebugODPSpecifiedRoute() != null) { + this.plDebugODPSpecifiedRoute = debuggeeSession.getPlDebugODPSpecifiedRoute(); + } // Debugger must connect to database host the same as debuggee - // 设置超时时间, 单位：us + // Set the timeout period, which is measured in microseconds (µs) List initSqls = Collections.singletonList( String.format("set session ob_query_timeout = %s;", DEBUG_TIMEOUT_MS * 1000)); acquireNewConnection(debuggeeSession.getConnectionSession(), () -> cloneDataSource(debuggeeSession.getNewDataSource(), initSqls)); try (Statement stmt = connection.createStatement()) { - // 绑定调试目标id - stmt.execute(String.format("call dbms_debug.attach_session(%s);", debuggeeSession.getDebugId())); + // Mount this new session of debugger to the previously initialized debuggee + stmt.execute(String.format("%s call dbms_debug.attach_session(%s);", PLUtils.getSpecifiedRoute( + this.plDebugODPSpecifiedRoute), debuggeeSession.getDebugId())); } catch (Exception e) { log.warn("Call dbms_debug.attach_session() failed", e); throw OBException @@ -182,7 +186,7 @@ public DebuggerSession(DebuggeeSession debuggeeSession, StartPLDebugReq req, boo private DebugDataSource cloneDataSource(DebugDataSource originDataSource, List initSqls) { ConnectionConfig config = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(connectionSession); - DebugDataSource debuggerDataSource = new DebugDataSource(config, initSqls); + DebugDataSource debuggerDataSource = new DebugDataSource(config, initSqls, this.plDebugODPSpecifiedRoute); debuggerDataSource.setUrl(originDataSource.getUrl()); debuggerDataSource.setUsername(originDataSource.getUsername()); debuggerDataSource.setPassword(originDataSource.getPassword()); @@ -247,7 +251,8 @@ public boolean detectSessionAlive() { boolean alive = false; try (Statement stmt = connection.createStatement()) { // 探测PL对象执行线程是否存活 - stmt.execute("select dbms_debug.target_program_running() from dual"); + stmt.execute(PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute) + + "select dbms_debug.target_program_running() from dual"); try (ResultSet resultSet = stmt.getResultSet()) { if (resultSet.next()) { alive = resultSet.getBoolean(1); @@ -369,7 +374,8 @@ public List getErrors() { dbplError.setType(debugType.name()); ConnectionConfig connectionConfig = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(connectionSession); - return getJdbcOperations().execute(new GetPLErrorCallBack(connectionConfig, dbplError)); + return getJdbcOperations() + .execute(new GetPLErrorCallBack(connectionConfig, dbplError, this.plDebugODPSpecifiedRoute)); } public List setBreakpoints(List breakpoints) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java index 6f2eb7a732..4e15e42484 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java @@ -38,6 +38,7 @@ import com.oceanbase.odc.service.pldebug.model.PLDebugResult; import com.oceanbase.odc.service.pldebug.model.PLDebugVariable; import com.oceanbase.odc.service.pldebug.model.StartPLDebugReq; +import com.oceanbase.odc.service.pldebug.util.PLUtils; import lombok.Data; import lombok.Setter; @@ -80,7 +81,8 @@ public PLDebugSession(long userId, IdGenerator idGenerator) { public void run() { if (Objects.nonNull(debuggerSession) && debuggerSession.detectSessionAlive()) { try (Statement stmt = debuggerSession.getConnection().createStatement()) { - stmt.execute("CALL DBMS_DEBUG.PING()"); + stmt.execute(PLUtils.getSpecifiedRoute(debuggerSession.getPlDebugODPSpecifiedRoute()) + + "CALL DBMS_DEBUG.PING()"); } catch (Exception e) { log.debug("Failed to call DBMS_DEBUG.PING()", e); } @@ -242,7 +244,8 @@ private void closeDebuggee() throws Exception { private void detach() { try (Statement stmt = debuggerSession.getConnection().createStatement()) { - stmt.execute("call dbms_debug.detach_session();"); + stmt.execute(PLUtils.getSpecifiedRoute(debuggerSession.getPlDebugODPSpecifiedRoute()) + + "call dbms_debug.detach_session();"); } catch (Exception e) { log.warn("fail to detach session on debugger", e); } @@ -250,7 +253,8 @@ private void detach() { private void debugOff() { try (Statement stmt = debuggeeSession.getConnection().createStatement()) { - stmt.execute("call dbms_debug.debug_off();"); + stmt.execute(PLUtils.getSpecifiedRoute(debuggeeSession.getPlDebugODPSpecifiedRoute()) + + "call dbms_debug.debug_off();"); } catch (Exception e) { log.warn("fail to debug off on debuggee", e); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/CallProcedureCallBack.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/CallProcedureCallBack.java index 2cd5cba4fb..a0319a5ec0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/CallProcedureCallBack.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/CallProcedureCallBack.java @@ -31,6 +31,7 @@ import com.oceanbase.odc.core.shared.exception.BadArgumentException; import com.oceanbase.odc.core.sql.util.DBPLObjectUtil; import com.oceanbase.odc.core.sql.util.JdbcDataTypeUtil; +import com.oceanbase.odc.service.pldebug.model.PLDebugODPSpecifiedRoute; import com.oceanbase.tools.dbbrowser.model.DBPLParam; import com.oceanbase.tools.dbbrowser.model.DBPLParamMode; import com.oceanbase.tools.dbbrowser.model.DBProcedure; @@ -50,6 +51,7 @@ public class CallProcedureCallBack implements ConnectionCallback private final DBProcedure procedure; private final SqlBuilder sqlBuilder; private final int timeoutSeconds; + private final PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute; public CallProcedureCallBack(@NonNull DBProcedure procedure, int timeoutSeconds, @NonNull SqlBuilder sqlBuilder) { @@ -58,10 +60,23 @@ public CallProcedureCallBack(@NonNull DBProcedure procedure, this.procedure = procedure; this.sqlBuilder = sqlBuilder; this.timeoutSeconds = timeoutSeconds; + this.plDebugODPSpecifiedRoute = null; + } + + public CallProcedureCallBack(@NonNull DBProcedure procedure, + int timeoutSeconds, @NonNull SqlBuilder sqlBuilder, + @NonNull PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute) { + Validate.notBlank(procedure.getProName(), "Procedure name can not be blank"); + DBPLObjectUtil.checkParams(procedure); + this.procedure = procedure; + this.sqlBuilder = sqlBuilder; + this.timeoutSeconds = timeoutSeconds; + this.plDebugODPSpecifiedRoute = plDebugODPSpecifiedRoute; } @Override public List doInConnection(Connection con) throws SQLException, DataAccessException { + sqlBuilder.append(PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute)); sqlBuilder.append("CALL "); if (StringUtils.isNotBlank(procedure.getPackageName())) { sqlBuilder.identifier(procedure.getPackageName()).append("."); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/OBOracleCallFunctionCallBack.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/OBOracleCallFunctionCallBack.java index 76fe37b804..f055690212 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/OBOracleCallFunctionCallBack.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/OBOracleCallFunctionCallBack.java @@ -31,6 +31,7 @@ import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.sql.util.DBPLObjectUtil; import com.oceanbase.odc.core.sql.util.JdbcDataTypeUtil; +import com.oceanbase.odc.service.pldebug.model.PLDebugODPSpecifiedRoute; import com.oceanbase.tools.dbbrowser.model.DBFunction; import com.oceanbase.tools.dbbrowser.model.DBPLParam; import com.oceanbase.tools.dbbrowser.model.DBPLParamMode; @@ -53,16 +54,29 @@ public class OBOracleCallFunctionCallBack implements ConnectionCallback params = new ArrayList<>(); if (function.getParams() != null) { @@ -101,8 +115,15 @@ public DBFunction doInConnection(Connection con) throws SQLException, DataAccess p.setDataType(function.getReturnType()); params.add(p); proc.setParams(params); - CallProcedureCallBack callBack = - new CallProcedureCallBack(proc, timeoutSeconds, new OracleSqlBuilder()); + CallProcedureCallBack callBack; + if (this.plDebugODPSpecifiedRoute == null) { + callBack = + new CallProcedureCallBack(proc, timeoutSeconds, new OracleSqlBuilder()); + } else { + callBack = + new CallProcedureCallBack(proc, timeoutSeconds, new OracleSqlBuilder(), + this.plDebugODPSpecifiedRoute); + } List callResult = callBack.doInConnection(con); if (CollectionUtils.isEmpty(callResult)) { return function; @@ -114,7 +135,7 @@ public DBFunction doInConnection(Connection con) throws SQLException, DataAccess return function; } finally { try (Statement stmt = con.createStatement()) { - stmt.execute("DROP PROCEDURE " + plName); + stmt.execute(PLUtils.getSpecifiedRoute(plDebugODPSpecifiedRoute) + "DROP PROCEDURE " + plName); } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLDebugTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLDebugTask.java index 46ede5b407..9b30a15255 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLDebugTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLDebugTask.java @@ -62,7 +62,8 @@ public PLDebugResult call() { PLDebugResult result = null; JdbcOperations jdbcOperations = debuggeeSession.getJdbcOperations(); if (anonymousBlock != null) { - jdbcOperations.execute(anonymousBlock); + jdbcOperations + .execute(PLUtils.getSpecifiedRoute(debuggeeSession.getPlDebugODPSpecifiedRoute()) + anonymousBlock); result = new PLDebugResult(); } log.info("Ending debug running, debugId={}", debugId); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLUtils.java index 0c8c4d80b1..e0e32cf2a6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLUtils.java @@ -18,6 +18,9 @@ import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.pldebug.model.PLDebugODPSpecifiedRoute; +import com.oceanbase.tools.dbbrowser.util.OracleSqlBuilder; +import com.oceanbase.tools.dbbrowser.util.SqlBuilder; /** * @author yaobin @@ -35,4 +38,17 @@ public static boolean isSys(ConnectionConfig connectionConfig) { return "SYS".equalsIgnoreCase(connectionConfig.getUsername()); } + + public static String getSpecifiedRoute(PLDebugODPSpecifiedRoute pLDebugODPSpecifiedRoute) { + SqlBuilder sqlBuilder = new OracleSqlBuilder(); + if (pLDebugODPSpecifiedRoute != null && pLDebugODPSpecifiedRoute.getObserverHost() != null + && pLDebugODPSpecifiedRoute.getObserverPort() != null) { + sqlBuilder.append("/* TARGET_DB_SERVER = '"); + sqlBuilder.append(pLDebugODPSpecifiedRoute.getObserverHost()); + sqlBuilder.append(":"); + sqlBuilder.append(pLDebugODPSpecifiedRoute.getObserverPort()); + sqlBuilder.append("' */"); + } + return sqlBuilder.toString(); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java index c134756210..7363549976 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java @@ -165,6 +165,7 @@ private void initSession(ConnectionSession session) { ConnectionInfoUtil.initSessionVersion(session); ConnectionSessionUtil.setConsoleSessionResetFlag(session, false); ConnectionInfoUtil.initConsoleConnectionId(session); + ConnectionInfoUtil.initOdpVersionIfExists(session); ConnectionSessionUtil.setConnectionConfig(session, connectionConfig); ConnectionSessionUtil.setColumnAccessor(session, new DatasourceColumnAccessor(session)); if (StringUtils.isNotBlank(connectionConfig.getTenantName())) { From 99fe1f5e3f9d80116438c56c336c5e67aca49688 Mon Sep 17 00:00:00 2001 From: zhangxiao <140503120+PeachThinking@users.noreply.github.com> Date: Mon, 16 Dec 2024 17:47:54 +0800 Subject: [PATCH 051/118] fix(db-browser): failed to get table due to create extended stats in column in oracle mode (#4026) * fix failed to get table in oracle mode when there is a extended stats in column * fix generate incorrect column ddl when column type is bit --- .../tools/dbbrowser/model/DBTableColumn.java | 2 +- .../schema/oracle/OracleSchemaAccessor.java | 2 +- .../schema/OracleSchemaAccessorTest.java | 10 ++++++++++ .../src/test/resources/table/oracle/drop.sql | 2 ++ .../table/oracle/testTableColumnDDL.sql | 6 ++++++ .../DefaultDBStructureComparatorTest.java | 18 ++++++++++++++++++ .../structurecompare/obmysql/source_drop.sql | 1 + .../obmysql/source_schema_ddl.sql | 7 ++++++- .../structurecompare/obmysql/target_drop.sql | 3 ++- .../obmysql/target_schema_ddl.sql | 6 +++++- .../oceanbase/odc/common/util/StringUtils.java | 2 +- 11 files changed, 53 insertions(+), 6 deletions(-) diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTableColumn.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTableColumn.java index fe019fa42f..65865fdc1f 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTableColumn.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTableColumn.java @@ -122,7 +122,7 @@ public class DBTableColumn implements DBObject, DBObjectWarningDescriptor { private String collationName; /** - * MySQL special + * The generation column's expression */ private String genExpression; diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java index 9591d11fea..d3c2de4b8d 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java @@ -869,7 +869,7 @@ protected RowMapper listColumnsRowMapper() { tableColumn.setVirtual("YES".equalsIgnoreCase(rs.getString(OracleConstants.COL_VIRTUAL_COLUMN))); tableColumn.setDefaultValue("NULL".equals(defaultValue) ? null : defaultValue); if (tableColumn.getVirtual()) { - tableColumn.setGenExpression(rs.getString(OracleConstants.COL_DATA_DEFAULT)); + tableColumn.setGenExpression(defaultValue); } return tableColumn; }; diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/schema/OracleSchemaAccessorTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/schema/OracleSchemaAccessorTest.java index 87f98a3e1c..4e867b4f1c 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/schema/OracleSchemaAccessorTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/schema/OracleSchemaAccessorTest.java @@ -137,6 +137,16 @@ public void listTableColumns_Success() { Assert.assertEquals(13, columns.size()); } + @Test + public void listTableColumns_TestCreateExtendedStats_Success() { + String sql = "SELECT DBMS_STATS.CREATE_EXTENDED_STATS('" + getOracleSchema() + + "', 'TEST_EXTENDED_STATS_COL', '(X, Y)') FROM DUAL"; + jdbcTemplate.execute(sql); + List columns = + accessor.listTableColumns(getOracleSchema(), "TEST_EXTENDED_STATS_COL"); + Assert.assertEquals(3, columns.size()); + } + @Test public void listTableConstraint_TestPrimaryKey_Success() { List constraintListList = diff --git a/libs/db-browser/src/test/resources/table/oracle/drop.sql b/libs/db-browser/src/test/resources/table/oracle/drop.sql index a492143ad2..abc60f8e79 100644 --- a/libs/db-browser/src/test/resources/table/oracle/drop.sql +++ b/libs/db-browser/src/test/resources/table/oracle/drop.sql @@ -26,6 +26,8 @@ call DROPIFEXISTS_TABLE('TEST_INDEX_TYPE') call DROPIFEXISTS_TABLE('TEST_VIEW_TABLE') / call DROPIFEXISTS_TABLE('PART_HASH_TEST') +/ +call DROPIFEXISTS_TABLE('TEST_EXTENDED_STATS_COL') diff --git a/libs/db-browser/src/test/resources/table/oracle/testTableColumnDDL.sql b/libs/db-browser/src/test/resources/table/oracle/testTableColumnDDL.sql index 228f8c78b9..5fa93ab5ef 100644 --- a/libs/db-browser/src/test/resources/table/oracle/testTableColumnDDL.sql +++ b/libs/db-browser/src/test/resources/table/oracle/testTableColumnDDL.sql @@ -13,4 +13,10 @@ create table TEST_COL_DATA_TYPE( col12 interval year to month, col13 interval day to second ) +/ + +CREATE TABLE TEST_EXTENDED_STATS_COL( + "X" NUMBER(*,0), + "Y" NUMBER(*,0) +) / \ No newline at end of file diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/structurecompare/obmysql/DefaultDBStructureComparatorTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/structurecompare/obmysql/DefaultDBStructureComparatorTest.java index 1ea3abf8dd..eca8748d1d 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/structurecompare/obmysql/DefaultDBStructureComparatorTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/structurecompare/obmysql/DefaultDBStructureComparatorTest.java @@ -406,4 +406,22 @@ public void test_modifyPartitionType() { excepted.setChangeScript("-- Unsupported operation to modify table partition type\n"); Assert.assertEquals(excepted, actual); } + + @Test + public void test_bitColumnDefaultValue() { + List actuals = results.stream().filter( + item -> item.getDbObjectName().equals("bit_column_default_value")).collect(Collectors.toList()) + .get(0).getSubDBObjectComparisonResult().stream() + .filter(item -> item.getDbObjectType().equals(DBObjectType.COLUMN) + && item.getComparisonResult().equals(ComparisonResult.ONLY_IN_SOURCE)) + .collect( + Collectors.toList()); + + DBObjectComparisonResult expect = + new DBObjectComparisonResult(DBObjectType.COLUMN, "bit_data_1", sourceSchemaName, targetSchemaName); + expect.setComparisonResult(ComparisonResult.ONLY_IN_SOURCE); + expect.setChangeScript("ALTER TABLE `" + targetSchemaName + + "`.`bit_column_default_value` ADD COLUMN `bit_data_1` bit(1) DEFAULT b'0' NOT NULL;\n"); + Assert.assertEquals(expect, actuals.get(0)); + } } diff --git a/server/integration-test/src/test/resources/structurecompare/obmysql/source_drop.sql b/server/integration-test/src/test/resources/structurecompare/obmysql/source_drop.sql index 116e2a2deb..deb24ee8f6 100644 --- a/server/integration-test/src/test/resources/structurecompare/obmysql/source_drop.sql +++ b/server/integration-test/src/test/resources/structurecompare/obmysql/source_drop.sql @@ -12,5 +12,6 @@ drop table if exists `converse_to_partition_table`; drop table if exists `converse_to_non_partition_table`; drop table if exists `modify_partition_type`; drop table if exists `update_options`; +drop table if exists `bit_column_default_value`; diff --git a/server/integration-test/src/test/resources/structurecompare/obmysql/source_schema_ddl.sql b/server/integration-test/src/test/resources/structurecompare/obmysql/source_schema_ddl.sql index d1890582b9..2d18a64c17 100644 --- a/server/integration-test/src/test/resources/structurecompare/obmysql/source_schema_ddl.sql +++ b/server/integration-test/src/test/resources/structurecompare/obmysql/source_schema_ddl.sql @@ -92,4 +92,9 @@ PARTITION BY HASH(`id`) PARTITIONS 4; create table `update_options`( `c1` INT(11) NOT NULL, `c2` INT(11) NOT NULL -) CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin COMMENT = 'comment1'; \ No newline at end of file +) CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin COMMENT = 'comment1'; + +CREATE TABLE `bit_column_default_value` ( + `id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT 'auto increment id', + `bit_data_1` BIT(1) NOT NULL DEFAULT b'0' +); \ No newline at end of file diff --git a/server/integration-test/src/test/resources/structurecompare/obmysql/target_drop.sql b/server/integration-test/src/test/resources/structurecompare/obmysql/target_drop.sql index 8da7a94dea..cb4c11d3a4 100644 --- a/server/integration-test/src/test/resources/structurecompare/obmysql/target_drop.sql +++ b/server/integration-test/src/test/resources/structurecompare/obmysql/target_drop.sql @@ -11,4 +11,5 @@ drop table if exists `update_partition`; drop table if exists `converse_to_partition_table`; drop table if exists `converse_to_non_partition_table`; drop table if exists `modify_partition_type`; -drop table if exists `update_options`; \ No newline at end of file +drop table if exists `update_options`; +drop table if exists `bit_column_default_value`; \ No newline at end of file diff --git a/server/integration-test/src/test/resources/structurecompare/obmysql/target_schema_ddl.sql b/server/integration-test/src/test/resources/structurecompare/obmysql/target_schema_ddl.sql index 71ff5cb491..ffe6a83ef6 100644 --- a/server/integration-test/src/test/resources/structurecompare/obmysql/target_schema_ddl.sql +++ b/server/integration-test/src/test/resources/structurecompare/obmysql/target_schema_ddl.sql @@ -95,4 +95,8 @@ PARTITION BY KEY(`id`) PARTITIONS 4; create table `update_options`( `c1` INT(11) NOT NULL, `c2` INT(11) NOT NULL -) CHARACTER SET = gb18030 COLLATE = gb18030_chinese_ci COMMENT = 'comment2'; \ No newline at end of file +) CHARACTER SET = gb18030 COLLATE = gb18030_chinese_ci COMMENT = 'comment2'; + +CREATE TABLE `bit_column_default_value` ( + `id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT 'auto increment id' +); \ No newline at end of file diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java index 9b8ef520b6..fe4acd2cfa 100644 --- a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java @@ -173,7 +173,7 @@ public static void quoteColumnDefaultValuesForMySQL(DBTable table) { table.getColumns().forEach(column -> { String defaultValue = column.getDefaultValue(); if (StringUtils.isNotEmpty(defaultValue)) { - if (!isDefaultValueBuiltInFunction(column)) { + if (!isDefaultValueBuiltInFunction(column) && !DataTypeUtil.isBitType(column.getTypeName())) { column.setDefaultValue("'".concat(defaultValue.replace("'", "''")).concat("'")); } } From 6e6a4631d0ff8c254a988446392c1ae09403cdc2 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Mon, 16 Dec 2024 18:04:42 +0800 Subject: [PATCH 052/118] feat(permission): support for global project roles (#3820) * ComposedPermission implementation * add ProjectPermission implementation * add actions in @PreAuthenticate * fix * refactor * add ut cases * delete deprecated code * refactor check project roles * refactor ResourceRoleService * bugfix * delete unused code * fix ut * response to comments * fix ut --- .../collaboration/ProjectServiceTest.java | 9 +- .../service/flow/FlowInstanceServiceTest.java | 5 +- .../service/iam/ResourceRoleServiceTest.java | 2 +- .../DefaultLoginSecurityManager.java | 6 + .../MethodAuthorizedInterceptor.java | 19 ++- .../permission/ComposedPermission.java | 59 +++++++++ .../permission/PermissionProvider.java | 9 ++ .../PrivateConnectionPermission.java | 113 ---------------- .../permission/ProjectPermission.java | 77 +++++++++++ .../authority/ComposedPermissionTest.java | 122 ++++++++++++++++++ .../tool/TestPermissionProvider.java | 12 +- .../common/V_4_3_3_1__iam_permission.yaml | 78 +++++++++++ .../migrate/common/V_4_3_3_2__iam_role.yaml | 81 ++++++++++++ .../V_4_3_3_3__iam_role_permission.yaml | 66 ++++++++++ .../odc/config/DefaultAuthConfiguration.java | 8 +- .../collaboration/ProjectRepository.java | 6 + .../odc/metadb/iam/PermissionRepository.java | 9 ++ .../oceanbase/odc/metadb/iam/RoleEntity.java | 2 + .../odc/metadb/iam/UserRoleRepository.java | 18 +++ .../collaboration/project/ProjectService.java | 31 +++-- .../service/connection/ConnectionService.java | 1 + .../connection/database/DatabaseService.java | 13 +- .../connection/model/ConnectProperties.java | 14 +- .../datasecurity/SensitiveColumnService.java | 33 +++-- .../datasecurity/SensitiveRuleService.java | 21 ++- .../odc/service/flow/FlowInstanceService.java | 2 +- .../iam/GlobalResourceRoleService.java | 65 ++++++++++ .../odc/service/iam/PermissionService.java | 16 +++ .../iam/ProjectPermissionValidator.java | 15 ++- .../ResourceRoleBasedPermissionExtractor.java | 6 +- .../odc/service/iam/ResourceRoleService.java | 122 ++++++++++++++++-- .../service/iam/UserPermissionService.java | 1 - .../odc/service/iam/UserService.java | 19 --- .../odc/service/iam/auth/BaseAuthorizer.java | 25 ++++ .../service/iam/auth/ComposedAuthorizer.java | 47 +++++++ .../iam/auth/DefaultAuthorizationFacade.java | 30 ++--- .../service/iam/auth/DefaultAuthorizer.java | 27 +--- .../iam/auth/DefaultPermissionProvider.java | 15 ++- .../iam/auth/ResourceRoleAuthorizer.java | 28 +--- .../iam/model/UserGlobalResourceRole.java | 37 ++++++ .../service/iam/model/UserResourceRole.java | 2 + .../iam/util/GlobalResourceRoleUtil.java | 55 ++++++++ .../notification/NotificationService.java | 36 ++++-- .../database/DatabasePermissionService.java | 6 +- .../table/TablePermissionService.java | 6 +- 45 files changed, 1072 insertions(+), 302 deletions(-) create mode 100644 server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ComposedPermission.java delete mode 100644 server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PrivateConnectionPermission.java create mode 100644 server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ProjectPermission.java create mode 100644 server/odc-core/src/test/java/com/oceanbase/odc/core/authority/ComposedPermissionTest.java create mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__iam_permission.yaml create mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_2__iam_role.yaml create mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_role_permission.yaml create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ComposedAuthorizer.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserGlobalResourceRole.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/iam/util/GlobalResourceRoleUtil.java diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java index 899878ad9f..f40330ffd1 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java @@ -130,7 +130,8 @@ public void testCreateProject_Success() { public void testGetProject_Success() { Project saved = projectService.create(getProject()); Mockito.when( - resourceRoleService.listByResourceTypeAndId(Mockito.eq(ResourceType.ODC_PROJECT), Mockito.anyLong())) + resourceRoleService.listByResourceTypeAndResourceId(Mockito.eq(ResourceType.ODC_PROJECT), + Mockito.anyLong())) .thenReturn(listUserResourceRole(saved.getId())); Project actual = projectService.detail(saved.getId()); Assert.assertNotNull(actual); @@ -141,7 +142,8 @@ public void testGetProject_syncTimeIsNull() { Date syncTime = new Date(); Project saved = projectService.create(getProject()); Mockito.when( - resourceRoleService.listByResourceTypeAndId(Mockito.eq(ResourceType.ODC_PROJECT), Mockito.anyLong())) + resourceRoleService.listByResourceTypeAndResourceId(Mockito.eq(ResourceType.ODC_PROJECT), + Mockito.anyLong())) .thenReturn(listUserResourceRole(saved.getId())); createDatabase(saved.getId(), null); createDatabase(saved.getId(), syncTime); @@ -154,7 +156,8 @@ public void testGetProject_syncTimeNotNull() { Date syncTime = new Date(); Project saved = projectService.create(getProject()); Mockito.when( - resourceRoleService.listByResourceTypeAndId(Mockito.eq(ResourceType.ODC_PROJECT), Mockito.anyLong())) + resourceRoleService.listByResourceTypeAndResourceId(Mockito.eq(ResourceType.ODC_PROJECT), + Mockito.anyLong())) .thenReturn(listUserResourceRole(saved.getId())); createDatabase(saved.getId(), syncTime); createDatabase(saved.getId(), DateUtils.addDays(syncTime, 1)); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java index 1242ca481e..8f82c4c474 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java @@ -54,6 +54,7 @@ import com.oceanbase.odc.ServiceTestEnv; import com.oceanbase.odc.common.event.EventPublisher; import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.core.authority.SecurityManager; import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.constant.ConnectionVisibleScope; import com.oceanbase.odc.core.shared.constant.FlowStatus; @@ -181,6 +182,8 @@ public class FlowInstanceServiceTest extends ServiceTestEnv { private UserTaskInstanceCandidateRepository userTaskInstanceCandidateRepository; @MockBean private DBResourcePermissionHelper permissionHelper; + @Autowired + private SecurityManager securityManager; @Before public void setUp() { @@ -405,7 +408,7 @@ public void detail_unauthorized_expThrown() { FlowInstanceEntity entity = optional.get(); entity.setCreatorId(-1); flowInstanceRepository.saveAndFlush(entity); - + securityManager.login(null, null); thrown.expect(AccessDeniedException.class); flowInstanceService.detail(flowInstance.getId()); } diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/iam/ResourceRoleServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/iam/ResourceRoleServiceTest.java index 329ada996c..ca735cae75 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/iam/ResourceRoleServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/iam/ResourceRoleServiceTest.java @@ -81,7 +81,7 @@ public void testListByResourceId_Success() { .thenReturn(Optional.of(getResourceRoleEntity().get(0))); Mockito.when(authenticationFacade.currentOrganizationId()).thenReturn(1L); resourceRoleService.saveAll(Arrays.asList(getProjectOwner())); - int actual = resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, 1L).size(); + int actual = resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, 1L).size(); Assert.assertEquals(1, actual); } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/DefaultLoginSecurityManager.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/DefaultLoginSecurityManager.java index 636fd9a4bc..2d0bec901e 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/DefaultLoginSecurityManager.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/DefaultLoginSecurityManager.java @@ -216,4 +216,10 @@ public Permission getPermissionByResourceRoles(SecurityResource resource, Collec return this.permissionProvider.getPermissionByResourceRoles(resource, resourceRoles); } + @Override + public Permission getPermissionByActionsAndResourceRoles(SecurityResource resource, Collection actions, + Collection resourceRoles) { + return this.permissionProvider.getPermissionByActionsAndResourceRoles(resource, actions, resourceRoles); + } + } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/interceptor/MethodAuthorizedInterceptor.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/interceptor/MethodAuthorizedInterceptor.java index 92e3a636ee..83dc340a39 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/interceptor/MethodAuthorizedInterceptor.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/interceptor/MethodAuthorizedInterceptor.java @@ -110,19 +110,26 @@ private Permission beforeAuthentication(PreAuthenticate annotation, Object targe } } List actions = Arrays.asList(annotation.actions()); - List roles = Arrays.asList(annotation.hasAnyResourceRole()); - String role = annotation.hasResourceRole(); + List resourceRoles = Arrays.asList(annotation.hasAnyResourceRole()); + String resourceRole = annotation.hasResourceRole(); + + if (CollectionUtils.isNotEmpty(actions) + && (CollectionUtils.isNotEmpty(resourceRoles) || StringUtils.isNotBlank(resourceRole))) { + return getSecurityManager().getPermissionByActionsAndResourceRoles( + new DefaultSecurityResource(resourceId, resourceType), actions, resourceRoles); + } + if (CollectionUtils.isNotEmpty(actions)) { return getSecurityManager().getPermissionByActions(new DefaultSecurityResource(resourceId, resourceType), actions); - } else if (CollectionUtils.isNotEmpty(roles)) { + } else if (CollectionUtils.isNotEmpty(resourceRoles)) { return getSecurityManager().getPermissionByResourceRoles( new DefaultSecurityResource(resourceId, resourceType), - roles); - } else if (StringUtils.isNotBlank(role)) { + resourceRoles); + } else if (StringUtils.isNotBlank(resourceRole)) { return getSecurityManager().getPermissionByResourceRoles( new DefaultSecurityResource(resourceId, resourceType), - Collections.singleton(role)); + Collections.singleton(resourceRole)); } else { throw new NullPointerException("The actions and hasAnyRole are both empty"); } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ComposedPermission.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ComposedPermission.java new file mode 100644 index 0000000000..eb60b38435 --- /dev/null +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ComposedPermission.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.core.authority.permission; + +import java.util.List; + +import org.springframework.util.CollectionUtils; + +import lombok.Getter; +import lombok.NonNull; + +/** + * @Author: Lebie + * @Date: 2024/11/11 15:11 + * @Description: [] + */ +@Getter +public class ComposedPermission implements Permission { + private final List permissions; + + public ComposedPermission(@NonNull List permissions) { + this.permissions = permissions; + } + + @Override + public boolean implies(Permission permission) { + if (!(permission instanceof ComposedPermission)) { + return false; + } + ComposedPermission composedPermission = (ComposedPermission) permission; + if (CollectionUtils.isEmpty(composedPermission.getPermissions())) { + return true; + } + if (CollectionUtils.isEmpty(this.permissions)) { + return false; + } + for (Permission thatPermission : composedPermission.getPermissions()) { + for (Permission thisPermission : this.permissions) { + if (thisPermission.implies(thatPermission)) { + return true; + } + } + } + return false; + } +} diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PermissionProvider.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PermissionProvider.java index 5ec7097dcf..7593166b41 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PermissionProvider.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PermissionProvider.java @@ -48,4 +48,13 @@ public interface PermissionProvider { */ Permission getPermissionByResourceRoles(SecurityResource resource, Collection resourceRoles); + /** + * @param resource {@link SecurityResource} + * @param actions action collection + * @param resourceRoles, see {@link ResourceRoleName} enums + * @return permission collection + */ + Permission getPermissionByActionsAndResourceRoles(SecurityResource resource, Collection actions, + Collection resourceRoles); + } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PrivateConnectionPermission.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PrivateConnectionPermission.java deleted file mode 100644 index c0bc79cafc..0000000000 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PrivateConnectionPermission.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.core.authority.permission; - -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Validate; - -import com.oceanbase.odc.core.shared.constant.ResourceType; - -/** - * Permission for private Connection object - * - * @author yh263208 - * @date 2021-09-06 19:23 - * @since ODC_release_3.2.0 - * @see Permission - * @see ResourcePermission - */ -public class PrivateConnectionPermission extends ResourcePermission { - private static final long serialVersionUID = 7923423426638008112L; - - public static final String CONNECTION_USE = "use"; - - public static final int CONNECT = 0x20; - public static final int USE = ResourcePermission.READ | CONNECT; - public static final int ALL = ResourcePermission.ALL | CONNECT; - - public PrivateConnectionPermission(String resourceId, String action) { - super(resourceId, ResourceType.ODC_PRIVATE_CONNECTION.name(), action); - } - - @Override - protected int getMaskFromAction(String action) { - int mask = super.getMaskFromAction(action); - if (action == null || StringUtils.isBlank(action)) { - return mask; - } - String newAction = action.replaceAll(" |\r|\n|\f|\t", ""); - if ("*".equals(action)) { - return ALL; - } - String[] actionList = newAction.split(","); - for (String actionItem : actionList) { - if (CONNECTION_USE.equalsIgnoreCase(actionItem)) { - mask |= USE; - } else if ("*".equals(actionItem)) { - return ALL; - } - } - return mask; - } - - public static Set getAllActions() { - Set returnVal = ResourcePermission.getAllActions(); - returnVal.addAll(Collections.singletonList(CONNECTION_USE)); - return returnVal; - } - - public static String getActions(int mask) { - List actionList = getActionList(mask); - return String.join(",", actionList); - } - - public static List getActionList(int mask) { - List actionList = new LinkedList<>(); - if ((mask & ALL) == ALL) { - actionList.add("*"); - return actionList; - } - List actions = ResourcePermission.getActionList(mask); - if (CollectionUtils.containsAny(actions, "*")) { - actionList.addAll(ResourcePermission.getAllActions()); - } else { - actionList.addAll(actions); - } - if ((mask & USE) == USE) { - actionList.add(CONNECTION_USE); - } - return actionList; - } - - @Override - protected void initPermissionMask(int mask) { - Validate.isTrue((mask & ALL) == mask, "Mask value is illegal"); - this.mask = mask; - } - - @Override - public String toString() { - return ResourceType.ODC_PRIVATE_CONNECTION.getLocalizedMessage() - + ":" + this.resourceId + ": " + getActions(this.mask); - } - -} diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ProjectPermission.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ProjectPermission.java new file mode 100644 index 0000000000..dd45260e77 --- /dev/null +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ProjectPermission.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.core.authority.permission; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.Validate; + +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.core.authority.model.SecurityResource; + +import lombok.Getter; +import lombok.NonNull; + +/** + * @Author: Lebie + * @Date: 2024/11/11 19:26 + * @Description: [] + */ +@Getter +public class ProjectPermission extends ResourcePermission { + private final String resourceId; + private final String resourceType; + private final List actions; + + public ProjectPermission(@NonNull SecurityResource resource, String action) { + super(resource.resourceId(), resource.resourceType(), action); + this.resourceId = resource.resourceId(); + this.resourceType = resource.resourceType(); + Validate.notNull(resourceId, "ResourceId can not be null"); + Validate.notNull(resourceType, "ResourceType can not be null"); + Validate.notEmpty(action); + this.actions = Arrays.asList(StringUtils.split(action, ",")).stream().map(e -> e.trim().toUpperCase()) + .collect(Collectors.toList()); + } + + public ProjectPermission(@NonNull SecurityResource resource, List actions) { + super(resource.resourceId(), resource.resourceType(), actions.stream().collect(Collectors.joining(","))); + this.resourceId = resource.resourceId(); + this.resourceType = resource.resourceType(); + Validate.notNull(resourceId, "ResourceId can not be null"); + Validate.notNull(resourceType, "ResourceType can not be null"); + Validate.notEmpty(actions); + this.actions = actions; + } + + + @Override + public boolean implies(Permission permission) { + if (this == permission) { + return true; + } + if (!(permission instanceof ProjectPermission)) { + return false; + } + ProjectPermission that = (ProjectPermission) permission; + return (this.resourceId.equals(that.resourceId) || "*".equals(this.resourceId)) + && (this.resourceType.equals(that.resourceType) || "*".equals(this.resourceType)) + && !Collections.disjoint(this.actions, that.getActions()); + } +} diff --git a/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/ComposedPermissionTest.java b/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/ComposedPermissionTest.java new file mode 100644 index 0000000000..907b5990bf --- /dev/null +++ b/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/ComposedPermissionTest.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.core.authority; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import com.oceanbase.odc.core.authority.model.DefaultSecurityResource; +import com.oceanbase.odc.core.authority.permission.ComposedPermission; +import com.oceanbase.odc.core.authority.permission.Permission; +import com.oceanbase.odc.core.authority.permission.ProjectPermission; +import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; + +/** + * @Author: Lebie + * @Date: 2024/11/13 17:45 + * @Description: [] + */ +public class ComposedPermissionTest { + @Test + public void implies_HasResourcePermission_impliesTrue() { + ComposedPermission thisPermission = new ComposedPermission(Arrays.asList(getResourcePermission("*", "DBA"))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertTrue(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_HasResourceRolePermission_impliesTrue() { + ComposedPermission thisPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertTrue(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_HasBothPermissions_impliesTrue() { + ComposedPermission thisPermission = new ComposedPermission(Arrays.asList(getResourcePermission("*", "DBA"), + getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertTrue(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_HasWrongAndRightPermissions_impliesTrue() { + ComposedPermission thisPermission = new ComposedPermission(Arrays.asList(getResourcePermission("*", "OWNER"), + getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertTrue(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_ImpliesNoPermission_impliesFalse() { + ComposedPermission thisPermission = new ComposedPermission(Arrays.asList(getResourcePermission("*", "OWNER"))); + ComposedPermission thatPermission = new ComposedPermission(Collections.emptyList()); + Assert.assertTrue(thisPermission.implies(thatPermission)); + } + + + @Test + public void implies_HasWrongResourceRolePermission_impliesFalse() { + ComposedPermission thisPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.OWNER)))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertFalse(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_HasWrongResourcePermission_impliesFalse() { + ComposedPermission thisPermission = new ComposedPermission( + Arrays.asList(getResourcePermission("*", "OWNER"))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertFalse(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_HasNoPermission_impliesFalse() { + ComposedPermission thisPermission = new ComposedPermission(Collections.emptyList()); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertFalse(thisPermission.implies(thatPermission)); + } + + private Permission getResourceRolePermission(String resourceId, List resourceRoles) { + return new ResourceRoleBasedPermission(new DefaultSecurityResource(resourceId, "ODC_PROJECT"), + resourceRoles); + } + + private Permission getResourcePermission(String resourceId, String actions) { + return new ProjectPermission(new DefaultSecurityResource(resourceId, "ODC_PROJECT"), actions); + } +} diff --git a/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/tool/TestPermissionProvider.java b/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/tool/TestPermissionProvider.java index 39b556a89d..75b009ac7b 100644 --- a/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/tool/TestPermissionProvider.java +++ b/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/tool/TestPermissionProvider.java @@ -15,14 +15,15 @@ */ package com.oceanbase.odc.core.authority.tool; +import java.util.Arrays; import java.util.Collection; import com.oceanbase.odc.core.authority.model.SecurityResource; +import com.oceanbase.odc.core.authority.permission.ComposedPermission; import com.oceanbase.odc.core.authority.permission.ConnectionPermission; import com.oceanbase.odc.core.authority.permission.DatabasePermission; import com.oceanbase.odc.core.authority.permission.Permission; import com.oceanbase.odc.core.authority.permission.PermissionProvider; -import com.oceanbase.odc.core.authority.permission.PrivateConnectionPermission; import com.oceanbase.odc.core.authority.permission.ResourcePermission; import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; import com.oceanbase.odc.core.shared.constant.ResourceType; @@ -41,8 +42,6 @@ public class TestPermissionProvider implements PermissionProvider { public Permission getPermissionByActions(SecurityResource resource, Collection actions) { if (ResourceType.ODC_CONNECTION.name().equals(resource.resourceType())) { return new ConnectionPermission(resource.resourceId(), String.join(",", actions)); - } else if (ResourceType.ODC_PRIVATE_CONNECTION.name().equals(resource.resourceType())) { - return new PrivateConnectionPermission(resource.resourceId(), String.join(",", actions)); } else if (ResourceType.ODC_DATABASE.name().equals(resource.resourceType())) { return new DatabasePermission(resource.resourceId(), String.join(",", actions)); } @@ -54,5 +53,12 @@ public Permission getPermissionByResourceRoles(SecurityResource resource, Collec return new ResourceRoleBasedPermission(resource, String.join(",", resourceRoles)); } + @Override + public Permission getPermissionByActionsAndResourceRoles(SecurityResource resource, Collection actions, + Collection resourceRoles) { + return new ComposedPermission(Arrays.asList(getPermissionByActions(resource, actions), + getPermissionByResourceRoles(resource, resourceRoles))); + } + } diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__iam_permission.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__iam_permission.yaml new file mode 100644 index 0000000000..15b4fcd630 --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__iam_permission.yaml @@ -0,0 +1,78 @@ +kind: resource +version: v2 +templates: + - metadata: + allow_duplicate: false + table_name: iam_permission + unique_keys: ["action", "organization_id", "resource_identifier", "type"] + specs: + - column_name: id + default_value: 73 + data_type: java.lang.Long + - column_name: action + value: "OWNER" + - column_name: resource_identifier + value: "ODC_PROJECT:*" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: type + value: "SYSTEM" + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project owner permission" + - metadata: + allow_duplicate: false + table_name: iam_permission + unique_keys: ["action", "organization_id", "resource_identifier", "type"] + specs: + - column_name: id + default_value: 74 + data_type: java.lang.Long + - column_name: action + value: "DBA" + - column_name: resource_identifier + value: "ODC_PROJECT:*" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: type + value: "SYSTEM" + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project DBA permission" + - metadata: + allow_duplicate: false + table_name: iam_permission + unique_keys: ["action", "organization_id", "resource_identifier", "type"] + specs: + - column_name: id + default_value: 75 + data_type: java.lang.Long + - column_name: action + value: "SECURITY_ADMINISTRATOR" + - column_name: resource_identifier + value: "ODC_PROJECT:*" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: type + value: "SYSTEM" + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project Security Administrator permission" \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_2__iam_role.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_2__iam_role.yaml new file mode 100644 index 0000000000..db18fd9b26 --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_2__iam_role.yaml @@ -0,0 +1,81 @@ +kind: resource +version: v2 +templates: + - metadata: + allow_duplicate: false + table_name: iam_role + unique_keys: ["name", "organization_id"] + specs: + - column_name: id + default_value: 5 + data_type: java.lang.Long + - column_name: name + value: "global_project_owner" + - column_name: type + value: "CUSTOM" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: is_enabled + value: true + data_type: java.lang.Boolean + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project owner, who is the owner of all projects" + - metadata: + allow_duplicate: false + table_name: iam_role + unique_keys: [ "name", "organization_id" ] + specs: + - column_name: id + default_value: 6 + data_type: java.lang.Long + - column_name: name + value: "global_project_dba" + - column_name: type + value: "CUSTOM" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: is_enabled + value: true + data_type: java.lang.Boolean + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project owner, who is the DBA of all projects" + - metadata: + allow_duplicate: false + table_name: iam_role + unique_keys: [ "name", "organization_id" ] + specs: + - column_name: id + default_value: 7 + data_type: java.lang.Long + - column_name: name + value: "global_project_security_administrator" + - column_name: type + value: "CUSTOM" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: is_enabled + value: true + data_type: java.lang.Boolean + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project owner, who is the security administrator of all projects" diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_role_permission.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_role_permission.yaml new file mode 100644 index 0000000000..f759c597ea --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_role_permission.yaml @@ -0,0 +1,66 @@ +kind: resource +version: v2 +templates: + - metadata: + allow_duplicate: false + table_name: iam_role_permission + unique_keys: [ "role_id", "permission_id" ] + specs: + - column_name: role_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_2__iam_role.yaml + field_path: templates.0.specs.0.value + - column_name: permission_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_1__iam_permission.yaml + field_path: templates.0.specs.0.value + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - metadata: + allow_duplicate: false + table_name: iam_role_permission + unique_keys: [ "role_id", "permission_id" ] + specs: + - column_name: role_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_2__iam_role.yaml + field_path: templates.1.specs.0.value + - column_name: permission_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_1__iam_permission.yaml + field_path: templates.1.specs.0.value + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - metadata: + allow_duplicate: false + table_name: iam_role_permission + unique_keys: [ "role_id", "permission_id" ] + specs: + - column_name: role_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_2__iam_role.yaml + field_path: templates.2.specs.0.value + - column_name: permission_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_1__iam_permission.yaml + field_path: templates.2.specs.0.value + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long \ No newline at end of file diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java index b74408bdb1..ef884b1598 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java @@ -23,6 +23,7 @@ import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.iam.ResourcePermissionExtractor; import com.oceanbase.odc.service.iam.ResourceRoleBasedPermissionExtractor; +import com.oceanbase.odc.service.iam.auth.ComposedAuthorizer; import com.oceanbase.odc.service.iam.auth.DefaultAuthorizer; import com.oceanbase.odc.service.iam.auth.ResourceRoleAuthorizer; @@ -40,8 +41,11 @@ public class DefaultAuthConfiguration extends BaseAuthConfiguration { protected Collection authorizers(PermissionRepository permissionRepository, ResourcePermissionExtractor resourcePermissionExtractor, UserResourceRoleRepository resourceRoleRepository, ResourceRoleBasedPermissionExtractor resourceRoleBasedPermissionExtractor) { - return Arrays.asList(new DefaultAuthorizer(permissionRepository, resourcePermissionExtractor), - new ResourceRoleAuthorizer(resourceRoleRepository, resourceRoleBasedPermissionExtractor)); + DefaultAuthorizer defaultAuthorizer = new DefaultAuthorizer(permissionRepository, resourcePermissionExtractor); + ResourceRoleAuthorizer resourceRoleAuthorizer = + new ResourceRoleAuthorizer(resourceRoleRepository, resourceRoleBasedPermissionExtractor); + ComposedAuthorizer composedAuthorizer = new ComposedAuthorizer(defaultAuthorizer, resourceRoleAuthorizer); + return Arrays.asList(defaultAuthorizer, resourceRoleAuthorizer, composedAuthorizer); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/collaboration/ProjectRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/collaboration/ProjectRepository.java index 1d74410ffc..a4df551c82 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/collaboration/ProjectRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/collaboration/ProjectRepository.java @@ -21,6 +21,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; public interface ProjectRepository extends JpaRepository, JpaSpecificationExecutor { Optional findByIdAndOrganizationId(Long id, Long organizationId); @@ -34,4 +36,8 @@ public interface ProjectRepository extends JpaRepository, J Optional findByUniqueIdentifier(String uniqueIdentifier); List findByUniqueIdentifierIn(Collection uniqueIdentifiers); + + @Query(value = "select p.id from collaboration_project p where p.organization_id = :organizationId", + nativeQuery = true) + List findIdsByOrganizationId(@Param("organizationId") Long organizationId); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java index c2a0856ec7..7b52bce20f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java @@ -97,6 +97,15 @@ Optional findByOrganizationIdAndActionAndResourceIdentifier(Lo nativeQuery = true) List findByExpireTimeBefore(@Param("expireTime") Date expireTime); + + @Query(value = "select p.* from iam_user_permission up inner join iam_permission p on up.permission_id=p.id " + + "where up.user_id=:userId and up.organization_id=:organizationId and p.resource_identifier=:resourceIdentifier " + + "and p.action in (:actions)", + nativeQuery = true) + List findByUserIdAndOrganizationIdAndResourceIdentifierAndActionIn(@Param("userId") Long userId, + @Param("organizationId") Long organizationId, @Param("resourceIdentifier") String resourceIdentifier, + @Param("actions") Collection actions); + @Modifying @Transactional @Query(value = "delete from iam_permission p where p.id in (:ids)", nativeQuery = true) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/RoleEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/RoleEntity.java index da59e2fa5c..6c3f50862b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/RoleEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/RoleEntity.java @@ -46,9 +46,11 @@ public class RoleEntity { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; + @Column(name = "name", nullable = false) private String name; @Enumerated(value = EnumType.STRING) + @Column(name = "type", nullable = false) private RoleType type; @Column(name = "is_enabled", nullable = false) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserRoleRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserRoleRepository.java index 7b9b5898ce..a5391053b6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserRoleRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserRoleRepository.java @@ -25,6 +25,8 @@ import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; +import com.oceanbase.odc.service.iam.model.UserGlobalResourceRole; + /** * @author gaoda.xy * @date 2022/12/5 19:20 @@ -42,6 +44,22 @@ public interface UserRoleRepository List findByOrganizationIdAndUserIdIn(Long organizationId, Collection userIds); + @Query("SELECT new com.oceanbase.odc.service.iam.model.UserGlobalResourceRole(ur.userId, r.name) " + + + "FROM RoleEntity r " + + "JOIN UserRoleEntity ur ON r.id = ur.roleId " + + "WHERE r.organizationId=:organizationId and r.name IN (:names)") + List findByOrganizationIdAndNameIn(@Param("organizationId") Long organizationId, + @Param("names") Collection names); + + @Query("SELECT new com.oceanbase.odc.service.iam.model.UserGlobalResourceRole(ur.userId, r.name) " + + + "FROM RoleEntity r " + + "JOIN UserRoleEntity ur ON r.id = ur.roleId " + + "WHERE r.organizationId=:organizationId and ur.userId=:userId and r.name IN (:names)") + List findByOrganizationIdAndUserIdAndNameIn(@Param("organizationId") Long organizationId, + @Param("userId") Long userId, @Param("names") Collection names); + @Modifying @Transactional @Query(value = "update iam_user_role set role_id=:roleId where id=:id", nativeQuery = true) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java index 9fb514ed8e..3b8deaf288 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java @@ -228,13 +228,14 @@ public Project create(@NotNull @Valid Project project) { } @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA", "DEVELOPER", "SECURITY_ADMINISTRATOR", "PARTICIPANT"}, + actions = {"OWNER", "DBA", "DEVELOPER", "PARTICIPANT", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public Project detail(@NotNull Long id) { ProjectEntity entity = repository.findByIdAndOrganizationId(id, currentOrganizationId()) .orElseThrow(() -> new NotFoundException(ResourceType.ODC_PROJECT, "id", id)); List userResourceRoles = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, entity.getId()); + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, entity.getId()); Project project = entityToModel(entity, userResourceRoles); project.setDbObjectLastSyncTime(getEarliestObjectSyncTime(id)); return project; @@ -245,7 +246,7 @@ public List getProjectMembers(@NotNull Long projectId, @NotNull L ProjectEntity entity = repository.findByIdAndOrganizationId(projectId, organizationId) .orElseThrow(() -> new NotFoundException(ResourceType.ODC_PROJECT, "id", projectId)); List userResourceRoles = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, entity.getId()); + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, entity.getId()); return userResourceRoles.stream().map(this::fromUserResourceRole).filter(Objects::nonNull) .collect(Collectors.toList()); } @@ -261,7 +262,8 @@ public Project getByIdentifier(@NotNull String uniqueIdentifier) { return entity.map(projectMapper::entityToModel).orElse(null); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public Project update(@NotNull Long id, @NotNull Project project) { ProjectEntity previous = repository.findByIdAndOrganizationId(id, currentOrganizationId()) @@ -278,10 +280,11 @@ public Project update(@NotNull Long id, @NotNull Project project) { previous.setName(project.getName()); ProjectEntity saved = repository.save(previous); return entityToModel(saved, - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, saved.getId())); + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, saved.getId())); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public Project setArchived(Long id, @NotNull SetArchivedReq req) throws InterruptedException { ProjectEntity saved = setArchived(id, currentOrganizationId(), req); @@ -325,7 +328,7 @@ public Page list(@Valid QueryProjectParams params, @NotNull Pageable pa Page projectEntities = innerList(params, pageable, UserResourceRole::isProjectMember); return projectEntities.map(project -> { List members = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, project.getId()); + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, project.getId()); return entityToModel(project, members); }); } @@ -353,7 +356,8 @@ private Page innerList(@Valid QueryProjectParams params, @NotNull return repository.findAll(specs, pageable); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public Project createProjectMembers(@NonNull Long id, @NotEmpty List members) { ProjectEntity project = repository.findByIdAndOrganizationId(id, currentOrganizationId()) @@ -386,7 +390,8 @@ public Project createMembersSkipPermissionCheck(@NonNull Long projectId, @NonNul } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public boolean deleteProjectMember(@NonNull Long projectId, @NonNull Long userId) { if (currentUserId().longValue() == userId.longValue()) { @@ -435,8 +440,9 @@ public TicketReference getProjectTicketReference(Long projectId) { @SkipAuthorize @Transactional(rollbackFor = Exception.class) public boolean deleteProjectMemberSkipPermissionCheck(@NonNull Long projectId, @NonNull Long userId) { - Set memberIds = resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, projectId).stream() - .filter(Objects::nonNull).map(UserResourceRole::getUserId).collect(Collectors.toSet()); + Set memberIds = + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, projectId).stream() + .filter(Objects::nonNull).map(UserResourceRole::getUserId).collect(Collectors.toSet()); if (!memberIds.contains(userId)) { throw new BadRequestException("User not belongs to this project"); } @@ -466,7 +472,8 @@ public void deleteUserRelatedProjectResources(@NonNull Long userId, @NonNull Str }); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public boolean updateProjectMember(@NonNull Long projectId, @NonNull Long userId, @NonNull List members) { @@ -483,7 +490,7 @@ public boolean updateProjectMemberSkipPermissionCheck(@NonNull Long projectId, @ ProjectEntity project = repository.findByIdAndOrganizationId(projectId, organizationId) .orElseThrow(() -> new NotFoundException(ResourceType.ODC_PROJECT, "id", projectId)); Map> userId2ResourceRoles = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, project.getId()).stream() + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, project.getId()).stream() .collect(Collectors.groupingBy(UserResourceRole::getUserId)); if (CollectionUtils.isEmpty(userId2ResourceRoles.keySet())) { return false; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java index 9278d7af65..60dcb7b5d1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java @@ -417,6 +417,7 @@ public List listSyncableDataSourcesByOrganizationIdIn(@NonNull @Transactional(rollbackFor = Exception.class) @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, DEVELOPER, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public PaginatedData listByProjectId(@NotNull Long projectId, @NotNull Boolean basic) { List connections; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index 26abc36750..cf78339e3c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -803,15 +803,17 @@ public Page listUserForOsc(Long dataSourceId) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public boolean modifyDatabasesOwners(@NotNull Long projectId, @NotNull @Valid ModifyDatabaseOwnerReq req) { databaseRepository.findByIdIn(req.getDatabaseIds()).forEach(database -> { if (!projectId.equals(database.getProjectId())) { throw new AccessDeniedException(); } }); - Set memberIds = resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, projectId).stream() - .map(UserResourceRole::getUserId).collect(Collectors.toSet()); + Set memberIds = + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, projectId).stream() + .map(UserResourceRole::getUserId).collect(Collectors.toSet()); if (!memberIds.containsAll(req.getOwnerIds())) { throw new AccessDeniedException(); } @@ -884,7 +886,8 @@ private void checkTransferable(@NonNull Collection databases, @N } if (CollectionUtils.isNotEmpty(req.getOwnerIds())) { Set memberIds = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, req.getProjectId()).stream() + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, req.getProjectId()) + .stream() .map(UserResourceRole::getUserId).collect(Collectors.toSet()); PreConditions.validArgumentState(memberIds.containsAll(req.getOwnerIds()), ErrorCodes.AccessDenied, null, "Invalid ownerIds"); @@ -935,7 +938,7 @@ private Page entitiesToModels(Page entities, boolean i Map> databaseId2UserResourceRole = new HashMap<>(); Map userId2User = new HashMap<>(); List userResourceRoles = - resourceRoleService.listByResourceTypeAndIdIn(ResourceType.ODC_DATABASE, databaseIds); + resourceRoleService.listByResourceTypeAndResourceIdIn(ResourceType.ODC_DATABASE, databaseIds); if (CollectionUtils.isNotEmpty(userResourceRoles)) { databaseId2UserResourceRole = userResourceRoles.stream() .collect(Collectors.groupingBy(UserResourceRole::getResourceId, Collectors.toList())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectProperties.java index 893c471017..33ad189c30 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectProperties.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectProperties.java @@ -25,8 +25,6 @@ import org.springframework.context.annotation.Configuration; import com.google.common.collect.Sets; -import com.oceanbase.odc.core.authority.permission.ConnectionPermission; -import com.oceanbase.odc.core.authority.permission.PrivateConnectionPermission; import lombok.Data; @@ -77,17 +75,7 @@ public Set getConnectionSupportedOperations(boolean temp, Set pe if (temp) { return getTempConnectionOperations(); } - if (CollectionUtils.isEmpty(permittedActions) - || permittedActions.contains(ConnectionPermission.CONNECTION_READWRITE) - || permittedActions.contains(PrivateConnectionPermission.CONNECTION_USE)) { - return getPersistentConnectionOperations(); - } - if (permittedActions.contains(ConnectionPermission.CONNECTION_READONLY)) { - Set operations = new HashSet<>(getPersistentConnectionOperations()); - operations.remove(DELETE_OPERATION); - return operations; - } - return NONE_OPERATIONS; + return getPersistentConnectionOperations(); } private Set getTempConnectionOperations() { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java index 45d737a979..b833d44f35 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java @@ -125,7 +125,8 @@ public class SensitiveColumnService { private static final SensitiveColumnMapper mapper = SensitiveColumnMapper.INSTANCE; @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public List listColumns(@NotNull Long projectId, @NotEmpty List databaseIds) { checkProjectDatabases(projectId, databaseIds); @@ -165,7 +166,8 @@ public List listColumns(@NotNull Long projectId, @NotEmp } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public Boolean exists(@NotNull Long projectId, @NotNull SensitiveColumn column) { PreConditions.notNull(column.getDatabase(), "database"); @@ -188,7 +190,8 @@ public boolean existsInCurrentOrganization() { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public List batchCreate(@NotNull Long projectId, @NotEmpty @Valid List columns) { @@ -220,7 +223,8 @@ public List batchCreate(@NotNull Long projectId, } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveColumn detail(@NotNull Long projectId, @NotNull Long id) { SensitiveColumnEntity entity = nullSafeGet(id); @@ -247,7 +251,8 @@ public SensitiveColumn detail(@NotNull Long projectId, @NotNull Long id) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public List batchUpdate(@NotNull Long projectId, @NotEmpty List ids, @NotNull Long maskingAlgorithmId) { @@ -265,7 +270,8 @@ public List batchUpdate(@NotNull Long projectId, @NotEmpty List } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public List batchDelete(@NotNull Long projectId, @NotEmpty List ids) { List entities = batchNullSafeGet(new HashSet<>(ids)); @@ -280,7 +286,8 @@ public List batchDelete(@NotNull Long projectId, @NotEmpty List } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public Page list(@NotNull Long projectId, @NotNull QuerySensitiveColumnParams params, Pageable pageable) { @@ -322,7 +329,8 @@ public Page list(@NotNull Long projectId, @NotNull QuerySensiti } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveColumnStats stats(@NotNull Long projectId) { SensitiveColumnStats stats = new SensitiveColumnStats(); @@ -348,7 +356,8 @@ public SensitiveColumnStats stats(@NotNull Long projectId) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveColumn setEnabled(@NotNull Long projectId, @NotNull Long id, @NotNull Boolean enabled) { SensitiveColumnEntity entity = nullSafeGet(id); @@ -363,7 +372,8 @@ public SensitiveColumn setEnabled(@NotNull Long projectId, @NotNull Long id, @No } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveColumnScanningTaskInfo startScanning(@NotNull Long projectId, @NotNull @Valid SensitiveColumnScanningReq req) { @@ -394,7 +404,8 @@ public SensitiveColumnScanningTaskInfo startScanning(@NotNull Long projectId, } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveColumnScanningTaskInfo getScanningResults(@NotNull Long projectId, @NotBlank String taskId) { SensitiveColumnScanningTaskInfo taskInfo = scanningTaskManager.get(taskId); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveRuleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveRuleService.java index 3174ca5ec1..0bc0fb9b05 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveRuleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveRuleService.java @@ -85,7 +85,8 @@ public class SensitiveRuleService { private static final SensitiveRuleMapper ruleMapper = SensitiveRuleMapper.INSTANCE; @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Boolean exists(@NotNull Long projectId, @NotBlank String name) { SensitiveRuleEntity entity = new SensitiveRuleEntity(); entity.setName(name); @@ -95,7 +96,8 @@ public Boolean exists(@NotNull Long projectId, @NotBlank String name) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveRule create(@NotNull Long projectId, @NotNull @Valid SensitiveRule rule) { Long organizationId = authenticationFacade.currentOrganizationId(); @@ -115,7 +117,8 @@ public SensitiveRule create(@NotNull Long projectId, @NotNull @Valid SensitiveRu } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveRule detail(@NotNull Long projectId, @NotNull Long id) { SensitiveRuleEntity entity = nullSafeGet(id); @@ -130,7 +133,8 @@ public SensitiveRule detail(@NotNull Long projectId, @NotNull Long id) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveRule update(@NotNull Long projectId, @NotNull Long id, @NotNull @Valid SensitiveRule rule) { SensitiveRuleEntity entity = nullSafeGet(id); @@ -162,7 +166,8 @@ public SensitiveRule update(@NotNull Long projectId, @NotNull Long id, @NotNull } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveRule delete(@NotNull Long projectId, @NotNull Long id) { SensitiveRuleEntity entity = nullSafeGet(id); @@ -179,7 +184,8 @@ public SensitiveRule delete(@NotNull Long projectId, @NotNull Long id) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public Page list(@NotNull Long projectId, @NotNull @Valid QuerySensitiveRuleParams params, Pageable pageable) { @@ -195,7 +201,8 @@ public Page list(@NotNull Long projectId, @NotNull @Valid QuerySe } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveRule setEnabled(@NotNull Long projectId, @NotNull Long id, @NotNull Boolean enabled) { SensitiveRuleEntity entity = nullSafeGet(id); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java index b8afd3fa7b..4e1985d504 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java @@ -1173,7 +1173,7 @@ private TemplateVariables buildTemplateVariables(CreateFlowInstanceReq flowInsta } List databaseOwners = new ArrayList<>(); List userResourceRoles = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_DATABASE, database.getId()); + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_DATABASE, database.getId()); if (CollectionUtils.isNotEmpty(userResourceRoles)) { Set userIds = userResourceRoles.stream().map(UserResourceRole::getUserId).collect(Collectors.toSet()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java new file mode 100644 index 0000000000..483167fdf8 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.metadb.iam.UserRoleRepository; +import com.oceanbase.odc.service.iam.model.UserGlobalResourceRole; +import com.oceanbase.odc.service.iam.util.GlobalResourceRoleUtil; + +/** + * @Author: Lebie + * @Date: 2024/11/19 15:47 + * @Description: [] + */ +@Service +public class GlobalResourceRoleService { + + @Autowired + private UserRoleRepository userRoleRepository; + + public List findGlobalResourceRoleUsersByOrganizationId(Long organizationId) { + return userRoleRepository.findByOrganizationIdAndNameIn( + organizationId, + Arrays.asList(GlobalResourceRoleUtil.GLOBAL_PROJECT_OWNER, GlobalResourceRoleUtil.GLOBAL_PROJECT_DBA, + GlobalResourceRoleUtil.GLOBAL_PROJECT_SECURITY_ADMINISTRATOR)); + } + + public List findGlobalResourceRoleUsersByOrganizationIdAndUserId(Long organizationId, + Long userId) { + return userRoleRepository.findByOrganizationIdAndUserIdAndNameIn(organizationId, userId, + Arrays.asList(GlobalResourceRoleUtil.GLOBAL_PROJECT_OWNER, GlobalResourceRoleUtil.GLOBAL_PROJECT_DBA, + GlobalResourceRoleUtil.GLOBAL_PROJECT_SECURITY_ADMINISTRATOR)); + } + + + public List findGlobalResourceRoleUsersByOrganizationIdAndRole(Long organizationId, + ResourceType resourceType, ResourceRoleName resourceRoleName) { + if (resourceType != ResourceType.ODC_PROJECT) { + return Collections.emptyList(); + } + return userRoleRepository.findByOrganizationIdAndNameIn( + organizationId, Arrays.asList(GlobalResourceRoleUtil.getGlobalRoleName(resourceRoleName))); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java index 24d42461f9..aba2e7c17d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java @@ -16,6 +16,8 @@ package com.oceanbase.odc.service.iam; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; @@ -121,4 +123,18 @@ public void deleteExpiredPermission(Date expiredTime) { } } + @SkipAuthorize + public List findGlobalResourceRolePermissions(Long userId, Long organizationId) { + return findByUserIdAndOrganizationIdAndResourceIdentifierAndActionIn(userId, organizationId, + "ODC_PROJECT:*", Arrays.asList("OWNER", "DBA", "SECURITY_ADMINISTRATOR")); + } + + + + private List findByUserIdAndOrganizationIdAndResourceIdentifierAndActionIn(Long userId, + Long organizationId, String resourceIdentifier, Collection actions) { + return permissionRepository.findByUserIdAndOrganizationIdAndResourceIdentifierAndActionIn(userId, + organizationId, resourceIdentifier, actions); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ProjectPermissionValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ProjectPermissionValidator.java index 7aed0759ab..42299ac19b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ProjectPermissionValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ProjectPermissionValidator.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.iam; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -25,7 +26,9 @@ import org.springframework.stereotype.Component; import com.oceanbase.odc.core.authority.model.DefaultSecurityResource; +import com.oceanbase.odc.core.authority.permission.ComposedPermission; import com.oceanbase.odc.core.authority.permission.Permission; +import com.oceanbase.odc.core.authority.permission.ProjectPermission; import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; @@ -73,9 +76,15 @@ public boolean hasProjectRole(@NonNull Collection projectIds, @NonNull Lis return false; } List permissions = projectIds.stream().filter(Objects::nonNull) - .map(projectId -> new ResourceRoleBasedPermission( - new DefaultSecurityResource(projectId.toString(), "ODC_PROJECT"), roleNames)) - .collect(Collectors.toList()); + .map(projectId -> { + Permission resourceRolePermission = new ResourceRoleBasedPermission( + new DefaultSecurityResource(projectId.toString(), "ODC_PROJECT"), roleNames); + Permission projectResourcePermission = + new ProjectPermission(new DefaultSecurityResource(projectId.toString(), "ODC_PROJECT"), + roleNames.stream().map(ResourceRoleName::name).collect( + Collectors.toList())); + return new ComposedPermission(Arrays.asList(resourceRolePermission, projectResourcePermission)); + }).collect(Collectors.toList()); return authorizationFacade.isImpliesPermissions(authenticationFacade.currentUser(), permissions); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleBasedPermissionExtractor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleBasedPermissionExtractor.java index 61e5f2bbd1..418ad3f632 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleBasedPermissionExtractor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleBasedPermissionExtractor.java @@ -49,11 +49,11 @@ public class ResourceRoleBasedPermissionExtractor { @Autowired private ResourceRoleRepository repository; - public List getResourcePermissions(List entities) { + public List getResourcePermissions(List entities) { if (CollectionUtils.isEmpty(entities)) { return Collections.EMPTY_LIST; } - List permissions = new ArrayList<>(); + List permissions = new ArrayList<>(); Map> resourceId2Entities = entities.stream().collect(Collectors.groupingBy(UserResourceRoleEntity::getResourceId)); for (Entry> entry : resourceId2Entities.entrySet()) { @@ -70,7 +70,7 @@ public List getResourcePermissions(List saveAll(List userResourceRoleLis public Set getResourceRoleIdentifiersByUserId(long organizationId, long userId) { List userResourceRoleEntities = userResourceRoleRepository.findByOrganizationIdAndUserId(organizationId, userId); - if (CollectionUtils.isEmpty(userResourceRoleEntities)) { - return Collections.emptySet(); - } - return userResourceRoleEntities.stream() + List globalResourceRolePermissions = + permissionService.findGlobalResourceRolePermissions(userId, organizationId); + + Set resourceRoleIdentifiers = userResourceRoleEntities.stream() .map(i -> StringUtils.join(i.getResourceId(), ":", i.getResourceRoleId())) .collect(Collectors.toSet()); + if (CollectionUtils.isEmpty(globalResourceRolePermissions)) { + return resourceRoleIdentifiers; + } + // Has global resource role + Map resourceRoleName2Id = resourceRoleRepository.findByResourceType(ResourceType.ODC_PROJECT) + .stream().map(resourceRoleMapper::entityToModel) + .collect(Collectors.toMap(role -> role.getRoleName().name(), ResourceRole::getId, (v1, v2) -> v2)); + + Set derivedFromGlobalResourceRole = globalResourceRolePermissions.stream() + .map(i -> StringUtils.join("*", ":", resourceRoleName2Id.get(i.getAction()))) + .collect(Collectors.toSet()); + derivedFromGlobalResourceRole.addAll(resourceRoleIdentifiers); + return derivedFromGlobalResourceRole; } @SkipAuthorize @@ -127,10 +152,25 @@ public Map> getProjectId2ResourceRoleNames(Long user Map id2ResourceRoles = resourceRoleRepository.findByResourceType(ResourceType.ODC_PROJECT) .stream().map(resourceRoleMapper::entityToModel) .collect(Collectors.toMap(ResourceRole::getId, resourceRole -> resourceRole, (v1, v2) -> v2)); - return userResourceRoleRepository + Map> result = userResourceRoleRepository .findByUserIdAndResourceTypeAndOrganizationId(userId, ResourceType.ODC_PROJECT, organizationId).stream() .collect(Collectors.groupingBy(UserResourceRoleEntity::getResourceId, Collectors.mapping( e -> id2ResourceRoles.get(e.getResourceRoleId()).getRoleName(), Collectors.toSet()))); + Set globalResourceRoles = + permissionService.findGlobalResourceRolePermissions(userId, organizationId).stream() + .map(i -> ResourceRoleName.valueOf(i.getAction())).collect(Collectors.toSet()); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return result; + } + projectRepository.findAllByOrganizationId(organizationId).stream().filter(p -> !p.getArchived()) + .forEach(p -> { + if (!result.containsKey(p.getId())) { + result.put(p.getId(), globalResourceRoles); + } else { + result.get(p.getId()).addAll(globalResourceRoles); + } + }); + return result; } @SkipAuthorize("internal usage") @@ -140,15 +180,39 @@ public Optional findResourceRoleById(@NonNull Long id) { @Transactional(rollbackFor = Exception.class) @SkipAuthorize("internal usage") - public List listByResourceTypeAndId(ResourceType resourceType, Long resourceId) { - return fromEntities(userResourceRoleRepository.listByResourceTypeAndId(resourceType, resourceId)); + public List listByResourceTypeAndResourceId(ResourceType resourceType, Long resourceId) { + List userResourceRoles = + fromEntities(userResourceRoleRepository.listByResourceTypeAndId(resourceType, resourceId)); + List globalResourceRoles = + globalResourceRoleService + .findGlobalResourceRoleUsersByOrganizationId(authenticationFacade.currentOrganizationId()); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return userResourceRoles; + } + globalResourceRoles.stream().map( + i -> new UserResourceRole(i.getUserId(), resourceId, ResourceType.ODC_PROJECT, i.getResourceRole(), + true)) + .forEach(userResourceRoles::add); + return userResourceRoles; } @Transactional(rollbackFor = Exception.class) @SkipAuthorize("internal usage") - public List listByResourceTypeAndIdIn(ResourceType resourceType, + public List listByResourceTypeAndResourceIdIn(ResourceType resourceType, @NotEmpty Collection resourceIds) { - return fromEntities(userResourceRoleRepository.listByResourceTypeAndIdIn(resourceType, resourceIds)); + List userResourceRoles = + fromEntities(userResourceRoleRepository.listByResourceTypeAndIdIn(resourceType, resourceIds)); + List globalResourceRoles = + globalResourceRoleService + .findGlobalResourceRoleUsersByOrganizationId(authenticationFacade.currentOrganizationId()); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return userResourceRoles; + } + globalResourceRoles.stream().flatMap(i -> resourceIds.stream().map( + resourceId -> new UserResourceRole(i.getUserId(), resourceId, ResourceType.ODC_PROJECT, + i.getResourceRole(), true))) + .forEach(userResourceRoles::add); + return userResourceRoles; } @Transactional(rollbackFor = Exception.class) @@ -185,19 +249,53 @@ public List listResourceRoles(List resourceType) { @SkipAuthorize("internal authenticated") public List listByOrganizationIdAndUserId(Long organizationId, Long userId) { - return fromEntities(userResourceRoleRepository.findByOrganizationIdAndUserId(organizationId, userId)); + List userResourceRoles = + fromEntities(userResourceRoleRepository.findByOrganizationIdAndUserId(organizationId, userId)); + List globalResourceRoles = + globalResourceRoleService.findGlobalResourceRoleUsersByOrganizationIdAndUserId(organizationId, userId); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return userResourceRoles; + } + projectRepository.findAllByOrganizationId(organizationId).stream().filter(p -> !p.getArchived()) + .forEach(p -> globalResourceRoles.stream() + .map(i -> new UserResourceRole(i.getUserId(), p.getId(), ResourceType.ODC_PROJECT, + i.getResourceRole(), true)) + .forEach(userResourceRoles::add)); + return userResourceRoles; } @SkipAuthorize("internal usage") public List listByUserId(Long userId) { - return fromEntities(userResourceRoleRepository.findByUserId(userId)); + List userResourceRoles = fromEntities(userResourceRoleRepository.findByUserId(userId)); + List globalResourceRoles = + globalResourceRoleService.findGlobalResourceRoleUsersByOrganizationIdAndUserId( + authenticationFacade.currentOrganizationId(), + userId); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return userResourceRoles; + } + projectRepository.findAllByOrganizationId(authenticationFacade.currentOrganizationId()).stream() + .filter(p -> !p.getArchived()).forEach(p -> globalResourceRoles.stream() + .map(i -> new UserResourceRole(i.getUserId(), p.getId(), ResourceType.ODC_PROJECT, + i.getResourceRole(), true)) + .forEach(userResourceRoles::add)); + return userResourceRoles; } @SkipAuthorize("internal usage") public List listByResourceIdAndTypeAndName(Long resourceId, ResourceType resourceType, String roleName) { - return fromEntities( + List userResourceRoles = fromEntities( userResourceRoleRepository.findByResourceIdAndTypeAndName(resourceId, resourceType, roleName)); + List globalResourceRoles = + globalResourceRoleService.findGlobalResourceRoleUsersByOrganizationIdAndRole( + authenticationFacade.currentOrganizationId(), resourceType, ResourceRoleName.valueOf(roleName)); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return userResourceRoles; + } + globalResourceRoles.stream().map(i -> new UserResourceRole(i.getUserId(), resourceId, resourceType, + i.getResourceRole(), true)).forEach(userResourceRoles::add); + return userResourceRoles; } private List fromEntities(Collection entities) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserPermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserPermissionService.java index f1aebab9a1..2a8756bc35 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserPermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserPermissionService.java @@ -148,5 +148,4 @@ public void bindUserAndDataSourcePermission(@NonNull Long userId, @NonNull Long }); } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java index 33e84a0220..9fc080e6f6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java @@ -25,7 +25,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -43,7 +42,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.Validate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.data.domain.Page; @@ -65,7 +63,6 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.base.MoreObjects; import com.oceanbase.odc.common.util.StringUtils; -import com.oceanbase.odc.core.authority.permission.Permission; import com.oceanbase.odc.core.authority.permission.ResourcePermission; import com.oceanbase.odc.core.authority.util.Authenticated; import com.oceanbase.odc.core.authority.util.PreAuthenticate; @@ -328,7 +325,6 @@ public User create(@NotNull @Valid CreateUserReq createUserReq) { log.debug("New user has been inserted, user: {}", userEntity); if (!Objects.isNull(createUserReq.getRoleIds()) && !createUserReq.getRoleIds().isEmpty()) { - inspectVerticalUnauthorized(authenticationFacade.currentUser(), createUserReq.getRoleIds()); for (Long roleId : createUserReq.getRoleIds()) { Role role = new Role(roleRepository.findById(roleId) .orElseThrow(() -> new NotFoundException(ResourceType.ODC_ROLE, "id", roleId))); @@ -748,7 +744,6 @@ public User update(long id, UpdateUserReq updateUserReq) { attachedRoleIds = relations.stream().map(UserRoleEntity::getRoleId).collect(Collectors.toSet()); } Long creatorId = authenticationFacade.currentUserId(); - inspectVerticalUnauthorized(authenticationFacade.currentUser(), updateUserReq.getRoleIds()); userRoleRepository.deleteByOrganizationIdAndUserId(authenticationFacade.currentOrganizationId(), id); userRoleRepository.flush(); for (Long roleId : updateUserReq.getRoleIds()) { @@ -925,19 +920,6 @@ public static class UserDeleteEvent { private String accountName; } - private void inspectVerticalUnauthorized(User operator, List roleIdsToBeAttached) { - Validate.notNull(roleIdsToBeAttached, - "RoleIdsToBeAttached can not be null for UserServcei#inspectVerticalUnauthorized"); - List permissions = new LinkedList<>( - permissionMapper.getResourcePermissions(permissionRepository.findByRoleIds(roleIdsToBeAttached))); - boolean checkResult = - authorizationFacade.isImpliesPermissions(operator, permissions); - if (!checkResult) { - String errMsg = "Cannot grant permissions that the current user does not have"; - throw new BadRequestException(ErrorCodes.GrantPermissionFailed, new Object[] {errMsg}, errMsg); - } - } - private List getUserIdsByRoleIds(@NonNull List roleIds) { List userIds = new ArrayList<>(); if (CollectionUtils.isEmpty(roleIds)) { @@ -1033,7 +1015,6 @@ private void batchBindRoles(List createUserReqs, Map(roleIds)); Map roleId2Entity = roleRepository.findByIdIn(roleIds).stream() .collect(Collectors.toMap(RoleEntity::getId, entity -> entity)); Verify.equals(roleIds.size(), roleId2Entity.keySet().size(), "roleIds.size()"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/BaseAuthorizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/BaseAuthorizer.java index e9d7fff075..df871b7d45 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/BaseAuthorizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/BaseAuthorizer.java @@ -17,6 +17,7 @@ import java.security.Principal; import java.util.Collection; +import java.util.List; import java.util.Objects; import com.oceanbase.odc.core.authority.auth.Authorizer; @@ -43,6 +44,28 @@ public void checkPermission(Principal principal, Collection permissi } } + @Override + public boolean isPermitted(Principal principal, Collection permissions, SecurityContext context) { + User odcUser = (User) principal; + if (Objects.isNull(odcUser.getId())) { + return false; + } + List permittedPermissions = listPermittedPermissions(principal); + for (Permission permission : permissions) { + boolean accessDenied = true; + for (Permission resourcePermission : permittedPermissions) { + if (resourcePermission.implies(permission)) { + accessDenied = false; + break; + } + } + if (accessDenied) { + return false; + } + } + return true; + } + /** * An Authorizer may not work for all Principal of the * Subject, so a method is needed to indicate whether a certain Authorizer @@ -55,4 +78,6 @@ public void checkPermission(Principal principal, Collection permissi public boolean supports(Class principal) { return User.class.isAssignableFrom(principal); } + + protected abstract List listPermittedPermissions(Principal principal); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ComposedAuthorizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ComposedAuthorizer.java new file mode 100644 index 0000000000..74e347f8e9 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ComposedAuthorizer.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.auth; + +import java.security.Principal; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.collections.ListUtils; + +import com.oceanbase.odc.core.authority.permission.ComposedPermission; +import com.oceanbase.odc.core.authority.permission.Permission; + +/** + * @Author: Lebie + * @Date: 2024/11/11 16:05 + * @Description: [] + */ +public class ComposedAuthorizer extends BaseAuthorizer { + private final DefaultAuthorizer defaultAuthorizer; + private final ResourceRoleAuthorizer resourceRoleAuthorizer; + + public ComposedAuthorizer(DefaultAuthorizer defaultAuthorizer, ResourceRoleAuthorizer resourceRoleAuthorizer) { + this.defaultAuthorizer = defaultAuthorizer; + this.resourceRoleAuthorizer = resourceRoleAuthorizer; + } + + @Override + protected List listPermittedPermissions(Principal principal) { + return Collections.singletonList( + new ComposedPermission(ListUtils.union(defaultAuthorizer.listPermittedPermissions(principal), + resourceRoleAuthorizer.listPermittedPermissions(principal)))); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java index b033bfb432..0ed97ce37a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java @@ -42,12 +42,13 @@ import org.springframework.data.jpa.domain.Specification; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import com.oceanbase.odc.core.authority.SecurityManager; import com.oceanbase.odc.core.authority.model.DefaultSecurityResource; import com.oceanbase.odc.core.authority.model.SecurityResource; import com.oceanbase.odc.core.authority.permission.ConnectionPermission; import com.oceanbase.odc.core.authority.permission.DatabasePermission; import com.oceanbase.odc.core.authority.permission.Permission; -import com.oceanbase.odc.core.authority.permission.PrivateConnectionPermission; +import com.oceanbase.odc.core.authority.permission.ProjectPermission; import com.oceanbase.odc.core.authority.permission.ResourcePermission; import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; import com.oceanbase.odc.core.shared.constant.PermissionType; @@ -86,6 +87,9 @@ public abstract class DefaultAuthorizationFacade implements AuthorizationFacade @Autowired @Qualifier("authorizationFacadeExecutor") private ThreadPoolTaskExecutor authorizationFacadeExecutor; + @Autowired + @Qualifier("servletSecurityManager") + private SecurityManager securityManager; @Override public Set getAllPermittedActions(Principal principal, ResourceType resourceType, String resourceId) { @@ -96,10 +100,7 @@ public Set getAllPermittedActions(Principal principal, ResourceType reso if (resourceType.equals(ResourceType.valueOf(((ResourcePermission) permission).getResourceType())) && ("*".equals(((ResourcePermission) permission).getResourceId()) || resourceId.equals(((ResourcePermission) permission).getResourceId()))) { - if (resourceType == ResourceType.ODC_PRIVATE_CONNECTION) { - returnVal.addAll( - PrivateConnectionPermission.getActionList(((ResourcePermission) permission).getMask())); - } else if (resourceType == ResourceType.ODC_CONNECTION) { + if (resourceType == ResourceType.ODC_CONNECTION) { returnVal.addAll( ConnectionPermission.getActionList(((ResourcePermission) permission).getMask())); } else if (resourceType == ResourceType.ODC_DATABASE) { @@ -165,10 +166,10 @@ public Map> getRelatedResourcesAndActions(Principa Set actions = returnVal.computeIfAbsent(resource, identifier -> new HashSet<>()); if (ResourceType.ODC_DATABASE.name().equals(resource.resourceType())) { actions.addAll(DatabasePermission.getActionList(((ResourcePermission) permission).getMask())); - } else if (ResourceType.ODC_PRIVATE_CONNECTION.name().equals(resource.resourceType())) { - actions.addAll(PrivateConnectionPermission.getActionList(((ResourcePermission) permission).getMask())); } else if (ResourceType.ODC_CONNECTION.name().equals(resource.resourceType())) { actions.addAll(ConnectionPermission.getActionList(((ResourcePermission) permission).getMask())); + } else if (ResourceType.ODC_PROJECT.name().equals(resource.resourceType())) { + actions.addAll(((ProjectPermission) permission).getActions()); } else { actions.addAll(ResourcePermission.getActionList(((ResourcePermission) permission).getMask())); } @@ -178,20 +179,7 @@ public Map> getRelatedResourcesAndActions(Principa @Override public boolean isImpliesPermissions(@NotNull Principal principal, @NotNull Collection permissions) { - List permittedPermissions = getAllPermissions(principal); - for (Permission permission : permissions) { - boolean implies = false; - for (Permission permittedPermission : permittedPermissions) { - if (permittedPermission.implies(permission)) { - implies = true; - break; - } - } - if (!implies) { - return false; - } - } - return true; + return this.securityManager.isPermitted(permissions); } private User entityToUser(UserEntity entity) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizer.java index 82f522b734..b831cd617f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizer.java @@ -16,13 +16,12 @@ package com.oceanbase.odc.service.iam.auth; import java.security.Principal; -import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; import com.oceanbase.odc.core.authority.auth.Authorizer; -import com.oceanbase.odc.core.authority.auth.SecurityContext; import com.oceanbase.odc.core.authority.permission.Permission; import com.oceanbase.odc.metadb.iam.PermissionEntity; import com.oceanbase.odc.metadb.iam.PermissionRepository; @@ -47,33 +46,15 @@ public DefaultAuthorizer(PermissionRepository repository, ResourcePermissionExtr this.permissionMapper = permissionMapper; } - // Todo 权限增加缓存 @Override - public boolean isPermitted(Principal principal, Collection permissions, SecurityContext context) { + protected List listPermittedPermissions(Principal principal) { User odcUser = (User) principal; if (Objects.isNull(odcUser.getId())) { - return false; + return Collections.emptyList(); } List permissionEntities = repository.findByUserIdAndUserStatusAndRoleStatusAndOrganizationId( odcUser.getId(), true, true, odcUser.getOrganizationId()).stream().filter(Objects::nonNull) .collect(Collectors.toList()); - if (permissionEntities.isEmpty()) { - return false; - } - Collection permissionCollection = - permissionMapper.getResourcePermissions(permissionEntities); - for (Permission permission : permissions) { - boolean accessDenied = true; - for (Permission resourcePermission : permissionCollection) { - if (resourcePermission.implies(permission)) { - accessDenied = false; - break; - } - } - if (accessDenied) { - return false; - } - } - return true; + return permissionMapper.getResourcePermissions(permissionEntities); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultPermissionProvider.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultPermissionProvider.java index 9b3e2a46bc..261d6521b8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultPermissionProvider.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultPermissionProvider.java @@ -15,14 +15,16 @@ */ package com.oceanbase.odc.service.iam.auth; +import java.util.Arrays; import java.util.Collection; import com.oceanbase.odc.core.authority.model.SecurityResource; +import com.oceanbase.odc.core.authority.permission.ComposedPermission; import com.oceanbase.odc.core.authority.permission.ConnectionPermission; import com.oceanbase.odc.core.authority.permission.DatabasePermission; import com.oceanbase.odc.core.authority.permission.Permission; import com.oceanbase.odc.core.authority.permission.PermissionProvider; -import com.oceanbase.odc.core.authority.permission.PrivateConnectionPermission; +import com.oceanbase.odc.core.authority.permission.ProjectPermission; import com.oceanbase.odc.core.authority.permission.ResourcePermission; import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; import com.oceanbase.odc.core.shared.constant.ResourceType; @@ -41,10 +43,10 @@ public class DefaultPermissionProvider implements PermissionProvider { public Permission getPermissionByActions(SecurityResource resource, Collection actions) { if (ResourceType.ODC_CONNECTION.name().equals(resource.resourceType())) { return new ConnectionPermission(resource.resourceId(), String.join(",", actions)); - } else if (ResourceType.ODC_PRIVATE_CONNECTION.name().equals(resource.resourceType())) { - return new PrivateConnectionPermission(resource.resourceId(), String.join(",", actions)); } else if (ResourceType.ODC_DATABASE.name().equals(resource.resourceType())) { return new DatabasePermission(resource.resourceId(), String.join(",", actions)); + } else if (ResourceType.ODC_PROJECT.name().equals(resource.resourceType())) { + return new ProjectPermission(resource, String.join(",", actions)); } return new ResourcePermission(resource, String.join(",", actions)); } @@ -54,4 +56,11 @@ public Permission getPermissionByResourceRoles(SecurityResource resource, Collec return new ResourceRoleBasedPermission(resource, String.join(",", resourceRoles)); } + @Override + public Permission getPermissionByActionsAndResourceRoles(SecurityResource resource, Collection actions, + Collection resourceRoles) { + return new ComposedPermission(Arrays.asList(getPermissionByActions(resource, actions), + getPermissionByResourceRoles(resource, resourceRoles))); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java index 3ba569ce31..25477cbbbe 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java @@ -16,14 +16,12 @@ package com.oceanbase.odc.service.iam.auth; import java.security.Principal; -import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -import com.oceanbase.odc.core.authority.auth.SecurityContext; import com.oceanbase.odc.core.authority.permission.Permission; -import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.iam.ResourceRoleBasedPermissionExtractor; @@ -44,14 +42,12 @@ public ResourceRoleAuthorizer(UserResourceRoleRepository repository, this.permissionMapper = permissionMapper; } - @Override - public boolean isPermitted(Principal principal, Collection permissions, SecurityContext context) { + protected List listPermittedPermissions(Principal principal) { User odcUser = (User) principal; if (Objects.isNull(odcUser.getId())) { - return false; + return Collections.emptyList(); } - /** * find all user-related resource role, and implies with permissions respectively */ @@ -59,22 +55,8 @@ public boolean isPermitted(Principal principal, Collection permissio repository.findByUserId(odcUser.getId()).stream() .filter(Objects::nonNull).collect(Collectors.toList()); if (resourceRoles.isEmpty()) { - return false; - } - Collection permissionCollection = - permissionMapper.getResourcePermissions(resourceRoles); - for (Permission permission : permissions) { - boolean accessDenied = true; - for (ResourceRoleBasedPermission resourceRoleBasedPermission : permissionCollection) { - if (resourceRoleBasedPermission.implies(permission)) { - accessDenied = false; - break; - } - } - if (accessDenied) { - return false; - } + return Collections.emptyList(); } - return true; + return permissionMapper.getResourcePermissions(resourceRoles); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserGlobalResourceRole.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserGlobalResourceRole.java new file mode 100644 index 0000000000..a785691968 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserGlobalResourceRole.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.model; + +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; +import com.oceanbase.odc.service.iam.util.GlobalResourceRoleUtil; + +import lombok.Data; + +/** + * @Author: Lebie + * @Date: 2024/11/19 17:15 + * @Description: [] + */ +@Data +public class UserGlobalResourceRole { + private Long userId; + private ResourceRoleName resourceRole; + + public UserGlobalResourceRole(Long userId, String globalRoleName) { + this.userId = userId; + this.resourceRole = GlobalResourceRoleUtil.getResourceRoleName(globalRoleName); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java index b2896050d8..83999e7539 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java @@ -44,6 +44,8 @@ public class UserResourceRole implements PermissionConfiguration { private ResourceRoleName resourceRole; + private boolean derivedFromGlobalProjectRole = false; + public boolean isProjectMember() { return this.resourceType == ResourceType.ODC_PROJECT && (this.resourceRole == ResourceRoleName.OWNER || this.resourceRole == ResourceRoleName.DBA diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/util/GlobalResourceRoleUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/util/GlobalResourceRoleUtil.java new file mode 100644 index 0000000000..22f955aeca --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/util/GlobalResourceRoleUtil.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.util; + +import java.util.HashMap; +import java.util.Map; + +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; + +/** + * @Author: Lebie + * @Date: 2024/11/19 16:05 + * @Description: [] + */ +public class GlobalResourceRoleUtil { + public final static String GLOBAL_PROJECT_OWNER = "global_project_owner"; + public final static String GLOBAL_PROJECT_DBA = "global_project_dba"; + public final static String GLOBAL_PROJECT_SECURITY_ADMINISTRATOR = "global_project_security_administrator"; + private final static Map globalRoleName2ResourceRoleName = new HashMap<>(); + private final static Map resourceRoleName2GlobalRoleName = new HashMap<>(); + + static { + globalRoleName2ResourceRoleName.put(GLOBAL_PROJECT_OWNER, ResourceRoleName.OWNER); + globalRoleName2ResourceRoleName.put(GLOBAL_PROJECT_DBA, ResourceRoleName.DBA); + globalRoleName2ResourceRoleName.put(GLOBAL_PROJECT_SECURITY_ADMINISTRATOR, + ResourceRoleName.SECURITY_ADMINISTRATOR); + + resourceRoleName2GlobalRoleName.put(ResourceRoleName.OWNER, GLOBAL_PROJECT_OWNER); + resourceRoleName2GlobalRoleName.put(ResourceRoleName.DBA, GLOBAL_PROJECT_DBA); + resourceRoleName2GlobalRoleName.put(ResourceRoleName.SECURITY_ADMINISTRATOR, + GLOBAL_PROJECT_SECURITY_ADMINISTRATOR); + } + + public static ResourceRoleName getResourceRoleName(String globalRoleName) { + return globalRoleName2ResourceRoleName.get(globalRoleName); + } + + public static String getGlobalRoleName(ResourceRoleName resourceRoleName) { + return resourceRoleName2GlobalRoleName.get(resourceRoleName); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationService.java index d3d630ac23..968d9c3247 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationService.java @@ -131,7 +131,8 @@ public void init() { NotificationPolicy::getPolicyMetadataId, policy -> policy, (p1, p2) -> p1, LinkedHashMap::new)); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Page listChannels(@NotNull Long projectId, @NotNull QueryChannelParams queryParams, @NotNull Pageable pageable) { Page channels = channelRepository.find(queryParams, pageable).map(channelMapper::fromEntity); @@ -140,13 +141,15 @@ public Page listChannels(@NotNull Long projectId, @NotNull QueryChannel } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Channel detailChannel(@NotNull Long projectId, @NotNull Long channelId) { return channelMapper.fromEntityWithConfig(nullSafeGetChannel(channelId, projectId)); } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Channel createChannel(@NotNull Long projectId, @NotNull Channel channel) { PreConditions.notBlank(channel.getName(), "channel.name"); PreConditions.notNull(channel.getType(), "channel.type"); @@ -166,7 +169,8 @@ public Channel createChannel(@NotNull Long projectId, @NotNull Channel channel) } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Channel updateChannel(@NotNull Long projectId, @NotNull Channel channel) { PreConditions.notNull(channel.getId(), "channel.id"); validator.validate(channel.getType(), channel.getChannelConfig()); @@ -189,7 +193,8 @@ public Channel updateChannel(@NotNull Long projectId, @NotNull Channel channel) } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Channel deleteChannel(@NotNull Long projectId, @NotNull Long channelId) { ChannelEntity entity = nullSafeGetChannel(channelId, projectId); @@ -200,7 +205,8 @@ public Channel deleteChannel(@NotNull Long projectId, @NotNull Long channelId) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public MessageSendResult testChannel(@NotNull Long projectId, @NotNull Channel channel) { PreConditions.notNull(channel.getType(), "channel.type"); PreConditions.notNull(channel.getChannelConfig(), "channel.config"); @@ -227,13 +233,15 @@ public MessageSendResult testChannel(@NotNull Long projectId, @NotNull Channel c } } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Boolean existsChannel(@NotNull Long projectId, @NotBlank String channelName) { Optional optional = channelRepository.findByProjectIdAndName(projectId, channelName); return optional.isPresent(); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List listPolicies(@NotNull Long projectId) { Map policies = new LinkedHashMap<>(metaPolicies); @@ -250,7 +258,8 @@ public List listPolicies(@NotNull Long projectId) { return new ArrayList<>(policies.values()); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public NotificationPolicy detailPolicy(@NotNull Long projectId, @NotNull Long policyId) { NotificationPolicyEntity entity = nullSafeGetPolicy(policyId); if (!Objects.equals(projectId, entity.getProjectId())) { @@ -263,7 +272,8 @@ public NotificationPolicy detailPolicy(@NotNull Long projectId, @NotNull Long po } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List batchUpdatePolicies(@NotNull Long projectId, @NotEmpty List policies) { List toBeCreated = new ArrayList<>(); @@ -282,13 +292,15 @@ public List batchUpdatePolicies(@NotNull Long projectId, return results; } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Page listMessages(@NotNull Long projectId, @NotNull QueryMessageParams queryParams, @NotNull Pageable pageable) { return messageRepository.find(queryParams, pageable).map(Message::fromEntity); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Message detailMessage(@NotNull Long projectId, @NotNull Long messageId) { Message message = Message.fromEntity(nullSafeGetMessage(messageId)); if (!Objects.equals(projectId, message.getProjectId())) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java index 4bf5a44542..d2ab1a8695 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java @@ -153,7 +153,8 @@ public Page list(@NotNull Long projectId, @NotNull Query } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List batchCreate(@NotNull Long projectId, @NotNull @Valid CreateDatabasePermissionReq req) { Set projectIds = projectService.getMemberProjectIds(req.getUserId()); @@ -220,7 +221,8 @@ public List batchCreate(@NotNull Long projectId, } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List batchRevoke(@NotNull Long projectId, @NotEmpty List ids) { List entities = userDatabasePermissionRepository.findByProjectIdAndIdIn(projectId, ids); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java index aa79900cad..9b8c04a465 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java @@ -167,7 +167,8 @@ public Page list(@NotNull Long projectId, @NotNull QueryTab } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List batchCreate(@NotNull Long projectId, @NotNull @Valid CreateTablePermissionReq req) { Set projectIds = projectService.getMemberProjectIds(req.getUserId()); @@ -242,7 +243,8 @@ public List batchCreate(@NotNull Long projectId, } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List batchRevoke(@NotNull Long projectId, @NotEmpty List ids) { List entities = userTablePermissionRepository.findByProjectIdAndIdIn(projectId, ids); From 4cdf2b1c310c58524ac3f21fcdec778539074032 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Tue, 17 Dec 2024 10:21:11 +0800 Subject: [PATCH 053/118] build: change code owners (#4033) * change code owners * change code owners * change --- .github/CODEOWNERS | 172 ++++++++++++++++++++++----------------------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ca7dbc75f0..94340c327d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,97 +1,97 @@ # ODC code owners, refer to https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners # These owners will be the default owners for everything in the repo. -* @yhilmare @yizhouxw +* @LioRoger @guowl3 @MarkPotato777 # libs -/libs/db-browser/ @yhilmare @PeachThinking -/libs/ob-sql-parser/ @yhilmare @PeachThinking +/libs/db-browser/ @PeachThinking @MarkPotato777 +/libs/ob-sql-parser/ @PeachThinking @MarkPotato777 # 3rd-party -/server/3rd-party/ @MarkPotato777 @yhilmare +/server/3rd-party/ @MarkPotato777 @LioRoger # common -/server/odc-common/ @yhilmare @yizhouxw +/server/odc-common/ @LioRoger @guowl3 # migrate -/server/odc-migrate/ @MarkPotato777 @yhilmare +/server/odc-migrate/ @MarkPotato777 @LioRoger # core -/server/odc-core/ @yhilmare @yizhouxw +/server/odc-core/ @LioRoger @guowl3 /server/odc-core/**/alarm/ @LuckyPickleZZ -/server/odc-core/**/authority/ @yhilmare +/server/odc-core/**/authority/ @MarkPotato777 /server/odc-core/**/datamasking/ @LuckyPickleZZ -/server/odc-core/**/datasource/ @yhilmare -/server/odc-core/**/flow/ @yhilmare -/server/odc-core/**/alarm/ @LuckyPickleZZ @yizhouxw -/server/odc-core/**/authority/ @yhilmare @yizhouxw -/server/odc-core/**/datamasking/ @LuckyPickleZZ @yhilmare -/server/odc-core/**/datasource/ @yhilmare @yizhouxw -/server/odc-core/**/flow/ @yhilmare @yizhouxw -/server/odc-core/**/migrate/ @yhilmare @yizhouxw -/server/odc-core/**/session/ @yhilmare @yizhouxw -/server/odc-core/**/shared/ @yhilmare @yizhouxw -/server/odc-core/**/sql/ @yhilmare @LuckyPickleZZ -/server/odc-core/**/task/ @yhilmare @yizhouxw @guowl3 +/server/odc-core/**/datasource/ @MarkPotato777 +/server/odc-core/**/flow/ @zijiacj @LioRoger @MarkPotato777 +/server/odc-core/**/alarm/ @LuckyPickleZZ @LioRoger +/server/odc-core/**/authority/ @MarkPotato777 @LioRoger +/server/odc-core/**/datamasking/ @LuckyPickleZZ @MarkPotato777 +/server/odc-core/**/datasource/ @MarkPotato777 @LioRoger +/server/odc-core/**/flow/ @zijiacj @LioRoger @MarkPotato777 +/server/odc-core/**/migrate/ @MarkPotato777 @LioRoger +/server/odc-core/**/session/ @LuckyPickleZZ @LioRoger +/server/odc-core/**/shared/ @LioRoger @guowl3 +/server/odc-core/**/sql/ @LuckyPickleZZ @LioRoger +/server/odc-core/**/task/ @LioRoger @guowl3 # service common -/server/odc-service/ @yhilmare @yizhouxw -/server/odc-service/**/config/ @yhilmare @yizhouxw -/server/odc-service/**/metadb/ @yhilmare @yizhouxw -/server/odc-service/**/service/common/ @yhilmare @MarkPotato777 +/server/odc-service/ @LioRoger @guowl3 +/server/odc-service/**/config/ @LioRoger @guowl3 +/server/odc-service/**/metadb/ @LioRoger @guowl3 +/server/odc-service/**/service/common/ @LioRoger @MarkPotato777 # service business -/server/odc-service/**/service/audit/ @MarkPotato777 @yizhouxw +/server/odc-service/**/service/audit/ @MarkPotato777 @LioRoger /server/odc-service/**/service/automation/ @LuckyPickleZZ @ungreat -/server/odc-service/**/service/captcha/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/collaboration/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/config/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/datasecurity/ @LuckyPickleZZ @yhilmare -/server/odc-service/**/service/datatransfer/ @LuckyPickleZZ @yhilmare -/server/odc-service/**/service/db/ @PeachThinking @yhilmare -/server/odc-service/**/service/diagnose/ @LuckyPickleZZ @yizhouxw -/server/odc-service/**/service/dispatch/ @yhilmare @yizhouxw +/server/odc-service/**/service/captcha/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/collaboration/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/config/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/datasecurity/ @LuckyPickleZZ @MarkPotato777 +/server/odc-service/**/service/datatransfer/ @LuckyPickleZZ @LioRoger +/server/odc-service/**/service/db/ @PeachThinking @LioRoger +/server/odc-service/**/service/diagnose/ @LuckyPickleZZ @LioRoger +/server/odc-service/**/service/dispatch/ @LioRoger @guowl3 /server/odc-service/**/service/dlm/ @guowl3 @kiko-art /server/odc-service/**/service/dml/ @LuckyPickleZZ @PeachThinking -/server/odc-service/**/service/encryption/ @PeachThinking @yizhouxw -/server/odc-service/**/service/feature/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/flow/ @yhilmare @yizhouxw +/server/odc-service/**/service/encryption/ @PeachThinking @LioRoger +/server/odc-service/**/service/feature/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/flow/ @zijiacj @LioRoger @MarkPotato777 /server/odc-service/**/service/i18n/ @LuckyPickleZZ -/server/odc-service/**/service/iam/ @MarkPotato777 @PeachThinking @yhilmare -/server/odc-service/**/service/info/ @yhilmare @yizhouxw -/server/odc-service/**/service/integration/ @yiminpeng @ungreat @yizhouxw +/server/odc-service/**/service/iam/ @MarkPotato777 @PeachThinking @LioRoger +/server/odc-service/**/service/info/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/integration/ @ungreat @LioRoger @yiminpeng /server/odc-service/**/service/lab/ @LuckyPickleZZ @ungreat -/server/odc-service/**/service/monitor/ @ungreat @yizhouxw +/server/odc-service/**/service/monitor/ @ungreat @ysjemmm @LioRoger /server/odc-service/**/service/notification/ @LuckyPickleZZ @MarkPotato777 -/server/odc-service/**/service/objectstorage/ @MarkPotato777 @yizhouxw +/server/odc-service/**/service/objectstorage/ @CHLK @MarkPotato777 /server/odc-service/**/service/onlineschemachange/ @LioRoger @LuckyPickleZZ -/server/odc-service/**/service/partitionplan/ @guowl3 @yhilmare -/server/odc-service/**/service/permission/ @MarkPotato777 @yhilmare -/server/odc-service/**/service/pldebug/ @yhilmare @yizhouxw -/server/odc-service/**/service/plugin/ @yhilmare @LuckyPickleZZ -/server/odc-service/**/service/quartz/ @guowl3 @yhilmare -/server/odc-service/**/service/requlation/ @MarkPotato777 @yhilmare -/server/odc-service/**/service/resourcegroup/ @MarkPotato777 @yhilmare +/server/odc-service/**/service/partitionplan/ @guowl3 @LioRoger +/server/odc-service/**/service/permission/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/pldebug/ @zijiacj @MarkPotato777 +/server/odc-service/**/service/plugin/ @LioRoger @LuckyPickleZZ +/server/odc-service/**/service/quartz/ @guowl3 @LioRoger +/server/odc-service/**/service/requlation/ @MarkPotato777 @zijiacj +/server/odc-service/**/service/resourcegroup/ @MarkPotato777 @zijiacj /server/odc-service/**/service/resultset/ @LuckyPickleZZ @PeachThinking /server/odc-service/**/service/rollbackplan/ @PeachThinking @MarkPotato777 -/server/odc-service/**/service/schedule/ @guowl3 @yhilmare -/server/odc-service/**/service/script/ @LuckyPickleZZ @yizhouxw -/server/odc-service/**/service/session/ @yhilmare @LuckyPickleZZ +/server/odc-service/**/service/schedule/ @guowl3 @LioRoger +/server/odc-service/**/service/script/ @LuckyPickleZZ @LioRoger +/server/odc-service/**/service/session/ @LuckyPickleZZ @LioRoger /server/odc-service/**/service/shadowtable/ @MarkPotato777 @PeachThinking -/server/odc-service/**/service/snippet/ @LuckyPickleZZ @yizhouxw -/server/odc-service/**/service/sqlcheck/ @yhilmare @PeachThinking -/server/odc-service/**/service/structurecompare/ @PeachThinking @yhilmare -/server/odc-service/**/service/systemconfig/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/task/ @yhilmare @guowl3 @yizhouxw -/server/odc-service/**/service/websocket/ @LuckyPickleZZ @yizhouxw +/server/odc-service/**/service/snippet/ @LuckyPickleZZ @LioRoger +/server/odc-service/**/service/sqlcheck/ @zijiacj @PeachThinking @MarkPotato777 +/server/odc-service/**/service/structurecompare/ @PeachThinking @MarkPotato777 +/server/odc-service/**/service/systemconfig/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/task/ @guowl3 @LioRoger +/server/odc-service/**/service/websocket/ @LuckyPickleZZ @LioRoger # plugins -/server/plugins/ @yhilmare @yizhouxw -/server/plugins/connect-plugin-doris/ @yhilmare @yizhouxw -/server/plugins/connect-plugin-mysql/ @yhilmare @yizhouxw -/server/plugins/connect-plugin-ob-mysql/ @yhilmare @yizhouxw -/server/plugins/connect-plugin-ob-oracle/ @yhilmare @yizhouxw -/server/plugins/connect-plugin-oracle/ @yhilmare @yizhouxw +/server/plugins/ @LioRoger @guowl3 @MarkPotato777 +/server/plugins/connect-plugin-doris/ @LioRoger @guowl3 @MarkPotato777 +/server/plugins/connect-plugin-mysql/ @LioRoger @guowl3 @MarkPotato777 +/server/plugins/connect-plugin-ob-mysql/ @LioRoger @guowl3 @MarkPotato777 +/server/plugins/connect-plugin-ob-oracle/ @LioRoger @guowl3 @MarkPotato777 +/server/plugins/connect-plugin-oracle/ @LioRoger @guowl3 @MarkPotato777 /server/plugins/schema-plugin-api/ @PeachThinking @MarkPotato777 /server/plugins/schema-plugin-doris/ @PeachThinking @MarkPotato777 @@ -101,37 +101,37 @@ /server/plugins/schema-plugin-odp-sharding-ob-mysql/ @PeachThinking @MarkPotato777 /server/plugins/schema-plugin-oracle/ @PeachThinking @MarkPotato777 -/server/plugins/task-plugin-api/ @LuckyPickleZZ @yhilmare -/server/plugins/task-plugin-doris/ @LuckyPickleZZ @yhilmare -/server/plugins/task-plugin-mysql/ @LuckyPickleZZ @yhilmare -/server/plugins/task-plugin-ob-mysql/ @LuckyPickleZZ @yhilmare -/server/plugins/task-plugin-ob-oracle/ @LuckyPickleZZ @yhilmare -/server/plugins/task-plugin-oracle/ @LuckyPickleZZ @yhilmare +/server/plugins/task-plugin-api/ @LuckyPickleZZ @LioRoger +/server/plugins/task-plugin-doris/ @LuckyPickleZZ @LioRoger +/server/plugins/task-plugin-mysql/ @LuckyPickleZZ @LioRoger +/server/plugins/task-plugin-ob-mysql/ @LuckyPickleZZ @LioRoger +/server/plugins/task-plugin-ob-oracle/ @LuckyPickleZZ @LioRoger +/server/plugins/task-plugin-oracle/ @LuckyPickleZZ @LioRoger # starters -/server/starters/ @yhilmare @yizhouxw -/server/starters/desktop-starter/ @yhilmare @yizhouxw -/server/starters/web-starter/ @yhilmare @MarkPotato777 +/server/starters/ @LioRoger @guowl3 @MarkPotato777 +/server/starters/desktop-starter/ @LioRoger @guowl3 @MarkPotato777 +/server/starters/web-starter/ @LioRoger @guowl3 @MarkPotato777 # modules -/server/modules/ @yhilmare @yizhouxw +/server/modules/ @LioRoger @CHLK # CI/CD -/.github/ @MarkPotato777 @yhilmare @yizhouxw -/builds/ @MarkPotato777 @yhilmare @yizhouxw -/script/ @MarkPotato777 @yhilmare @yizhouxw -/distribution/ @MarkPotato777 @yhilmare @yizhouxw -/server/odc-test/ @MarkPotato777 @yhilmare @yizhouxw -/server/integration-test/ @MarkPotato777 @yhilmare @yizhouxw -/server/test-script/ @MarkPotato777 @yhilmare @yizhouxw +/.github/ @MarkPotato777 @LioRoger +/builds/ @MarkPotato777 @LioRoger +/script/ @MarkPotato777 @LioRoger +/distribution/ @MarkPotato777 @LioRoger +/server/odc-test/ @MarkPotato777 @LioRoger +/server/integration-test/ @MarkPotato777 @LioRoger +/server/test-script/ @MarkPotato777 @LioRoger # i18n -/server/odc-core/src/main/resources/i18n/ @Jane201510 @JessieWuJiexi @yizhouxw +/server/odc-core/src/main/resources/i18n/ @Jane201510 @JessieWuJiexi @LioRoger @MarkPotato777 @guowl3 # docs -/docs/ @Jane201510 @yhilmare @yizhouxw -CHANGELOG.md @Jane201510 @JessieWuJiexi @yhilmare @yizhouxw -CHANGELOG-zh-CN.md @Jane201510 @JessieWuJiexi @yhilmare @yizhouxw -README.md @Jane201510 @JessieWuJiexi @yizhouxw -README-zh.md @Jane201510 @JessieWuJiexi @yizhouxw +/docs/ @Jane201510 @MarkPotato777 @LioRoger @guowl3 +CHANGELOG.md @Jane201510 @JessieWuJiexi @MarkPotato777 @LioRoger @guowl3 +CHANGELOG-zh-CN.md @Jane201510 @JessieWuJiexi @MarkPotato777 @LioRoger @guowl3 +README.md @Jane201510 @JessieWuJiexi @MarkPotato777 @LioRoger @guowl3 +README-zh.md @Jane201510 @JessieWuJiexi @MarkPotato777 @LioRoger @guowl3 From e974d37d93746694670a0040a0a494615fbd3e7b Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Tue, 17 Dec 2024 14:08:47 +0800 Subject: [PATCH 054/118] Update IAM role & permission migration files (#4041) --- ...ermission.yaml => V_4_3_3_3__iam_permission.yaml} | 0 ...3_3_2__iam_role.yaml => V_4_3_3_4__iam_role.yaml} | 0 ...sion.yaml => V_4_3_3_5__iam_role_permission.yaml} | 12 ++++++------ 3 files changed, 6 insertions(+), 6 deletions(-) rename server/odc-migrate/src/main/resources/migrate/common/{V_4_3_3_1__iam_permission.yaml => V_4_3_3_3__iam_permission.yaml} (100%) rename server/odc-migrate/src/main/resources/migrate/common/{V_4_3_3_2__iam_role.yaml => V_4_3_3_4__iam_role.yaml} (100%) rename server/odc-migrate/src/main/resources/migrate/common/{V_4_3_3_3__iam_role_permission.yaml => V_4_3_3_5__iam_role_permission.yaml} (82%) diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__iam_permission.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_permission.yaml similarity index 100% rename from server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__iam_permission.yaml rename to server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_permission.yaml diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_2__iam_role.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_4__iam_role.yaml similarity index 100% rename from server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_2__iam_role.yaml rename to server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_4__iam_role.yaml diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_role_permission.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_5__iam_role_permission.yaml similarity index 82% rename from server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_role_permission.yaml rename to server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_5__iam_role_permission.yaml index f759c597ea..952322539b 100644 --- a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_role_permission.yaml +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_5__iam_role_permission.yaml @@ -9,12 +9,12 @@ templates: - column_name: role_id value_from: field_ref: - ref_file: migrate/common/V_4_3_3_2__iam_role.yaml + ref_file: migrate/common/V_4_3_3_4__iam_role.yaml field_path: templates.0.specs.0.value - column_name: permission_id value_from: field_ref: - ref_file: migrate/common/V_4_3_3_1__iam_permission.yaml + ref_file: migrate/common/V_4_3_3_3__iam_permission.yaml field_path: templates.0.specs.0.value - column_name: organization_id value: ${ORGANIZATION_ID} @@ -30,12 +30,12 @@ templates: - column_name: role_id value_from: field_ref: - ref_file: migrate/common/V_4_3_3_2__iam_role.yaml + ref_file: migrate/common/V_4_3_3_4__iam_role.yaml field_path: templates.1.specs.0.value - column_name: permission_id value_from: field_ref: - ref_file: migrate/common/V_4_3_3_1__iam_permission.yaml + ref_file: migrate/common/V_4_3_3_3__iam_permission.yaml field_path: templates.1.specs.0.value - column_name: organization_id value: ${ORGANIZATION_ID} @@ -51,12 +51,12 @@ templates: - column_name: role_id value_from: field_ref: - ref_file: migrate/common/V_4_3_3_2__iam_role.yaml + ref_file: migrate/common/V_4_3_3_4__iam_role.yaml field_path: templates.2.specs.0.value - column_name: permission_id value_from: field_ref: - ref_file: migrate/common/V_4_3_3_1__iam_permission.yaml + ref_file: migrate/common/V_4_3_3_3__iam_permission.yaml field_path: templates.2.specs.0.value - column_name: organization_id value: ${ORGANIZATION_ID} From ef6ac07df23f278f7637d649a38c6048903ff817 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Tue, 17 Dec 2024 17:48:27 +0800 Subject: [PATCH 055/118] fix(permission):time is missing when initiating apply database task again (#4046) * Add expireTimeValue to ApplyDatabaseParameter for frontend task initiation * Refactor ApplyDatabaseParameter: rename expireTimeValue to validDuration --- .../permission/database/model/ApplyDatabaseParameter.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java index 6519a9e878..11f3bbb7bb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java @@ -47,6 +47,11 @@ public class ApplyDatabaseParameter implements Serializable, TaskParameters { * Permission types to be applied for, required */ private List types; + /** + * This field represents the duration of the permission. It is only used for the front end to + * re-initiate tasks, and the backend does not rely on it + */ + private String validDuration; /** * Expiration time, null means no expiration, optional */ From 47fee13372e9efb62ca6ed7c551b9029fa05a94f Mon Sep 17 00:00:00 2001 From: LioRoger Date: Thu, 19 Dec 2024 14:00:08 +0800 Subject: [PATCH 056/118] fix(osc): remove distinct from osc query user sql --- .../odc/service/onlineschemachange/ddl/OscOBMySqlAccessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/ddl/OscOBMySqlAccessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/ddl/OscOBMySqlAccessor.java index 290b387abf..f2a3a4ee63 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/ddl/OscOBMySqlAccessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/ddl/OscOBMySqlAccessor.java @@ -42,7 +42,7 @@ public List listUsers(List usernames) { MySQLSqlBuilder sb = new MySQLSqlBuilder(); // mysql.users in oceanbase do not provide account locked status. // so we query locked info from ocenbase.__all_user - sb.append("SELECT distinct user_name, is_locked, host FROM oceanbase.__all_user"); + sb.append("SELECT user_name, is_locked, host FROM oceanbase.__all_user"); if (CollectionUtils.isNotEmpty(usernames)) { sb.append(" WHERE user_name IN ("); sb.values(usernames); From 7f6db5d29bdfb8758e17726709610b2d786bfc79 Mon Sep 17 00:00:00 2001 From: guowl3 Date: Thu, 19 Dec 2024 15:54:13 +0800 Subject: [PATCH 057/118] opt offset unit (#4075) --- .../dlm/utils/DataArchiveConditionUtil.java | 35 +++++++------------ .../utils/DataArchiveConditionUtilTest.java | 9 +++-- 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtil.java index 1ee1f976b9..02babab202 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtil.java @@ -18,7 +18,6 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -30,6 +29,7 @@ import org.apache.commons.collections4.CollectionUtils; import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.core.shared.exception.UnsupportedException; import com.oceanbase.odc.service.dlm.model.OffsetConfig; import com.oceanbase.odc.service.dlm.model.Operator; @@ -82,21 +82,18 @@ private static String calculateDateTime(Date baseDate, OffsetConfig config) { if (StringUtils.isNotEmpty(config.getPattern())) { String[] parts = config.getPattern().split("\\|"); String offsetString = parts[1].substring(1); - ChronoUnit unit = parseUnit(offsetString.substring(offsetString.length() - 1)); - long offsetSeconds = parseValue(offsetString) * unit.getDuration().getSeconds(); + long offsetValue = parseValue(offsetString); if (parts[1].startsWith("-")) { - offsetSeconds = -offsetSeconds; + offsetValue = -offsetValue; } - localDateTime = baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime() - .plusSeconds(offsetSeconds); + localDateTime = calculateDateTime(baseDate, offsetValue, offsetString.substring(offsetString.length() - 1)); return localDateTime.format(DateTimeFormatter.ofPattern(parts[0])); } else { - long offsetSeconds = config.getValue() * parseUnit(config.getUnit()).getDuration().getSeconds(); + long offsetValue = config.getValue(); if (config.getOperator() == Operator.MINUS) { - offsetSeconds = -offsetSeconds; + offsetValue = -offsetValue; } - localDateTime = baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime() - .plusSeconds(offsetSeconds); + localDateTime = calculateDateTime(baseDate, offsetValue, config.getUnit()); return localDateTime.format(DateTimeFormatter.ofPattern(config.getDateFormatPattern())); } @@ -106,24 +103,18 @@ private static int parseValue(String offsetString) { return Integer.parseInt(offsetString.substring(0, offsetString.length() - 1)); } - private static ChronoUnit parseUnit(String unit) { + private static LocalDateTime calculateDateTime(Date baseDate, long offsetValue, String unit) { switch (unit) { case "y": - return ChronoUnit.YEARS; + return baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime().plusYears(offsetValue); case "M": - return ChronoUnit.MONTHS; + return baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime().plusMonths(offsetValue); case "d": - return ChronoUnit.DAYS; - case "h": - return ChronoUnit.HOURS; - case "m": - return ChronoUnit.MINUTES; - case "s": - return ChronoUnit.SECONDS; + return baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime().plusDays(offsetValue); case "w": - return ChronoUnit.WEEKS; + return baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime().plusWeeks(offsetValue); default: - throw new IllegalArgumentException("Unknown unit: " + unit); + throw new UnsupportedException("Unsupported unit: " + unit); } } } diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtilTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtilTest.java index 6c8dd523f7..bf2fdd21fb 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtilTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtilTest.java @@ -46,7 +46,7 @@ public void testParseCondition() { String condition = "select * from test where gmt_create>'${start}' and gmt_create<'${end}'"; OffsetConfig config = new OffsetConfig(); config.setName("start"); - config.setPattern("yyyy-MM-dd HH:mm:ss|+10m"); + config.setPattern("yyyy-MM-dd|+10M"); OffsetConfig config2 = new OffsetConfig(); config2.setName("end"); config2.setOperator(Operator.PLUS); @@ -58,12 +58,11 @@ public void testParseCondition() { variables.add(config2); String result = DataArchiveConditionUtil.parseCondition(condition, variables, date); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); Calendar calendar = Calendar.getInstance(); calendar.setTime(date); - calendar.add(Calendar.MINUTE, 10); + calendar.add(Calendar.MONTH, 10); String expect = condition.replace("${start}", sdf.format(calendar.getTime())); - sdf = new SimpleDateFormat("yyyy-MM-dd"); calendar.setTime(date); calendar.add(Calendar.DAY_OF_MONTH, 1); expect = expect.replace("${end}", sdf.format(calendar.getTime())); @@ -76,7 +75,7 @@ public void testNotFoundVariable() { String condition = "select * from test where gmt_create>'${start}' and gmt_create<'${end}'"; OffsetConfig config = new OffsetConfig(); config.setName("start"); - config.setPattern("yyyy-MM-dd HH:mm:ss|+10m"); + config.setPattern("yyyy-MM-dd HH:mm:ss|+10M"); List variables = new LinkedList<>(); variables.add(config); From 6422d017b5f083668def5c19cd1c6ee4556f2a6e Mon Sep 17 00:00:00 2001 From: guowl3 Date: Fri, 20 Dec 2024 17:42:57 +0800 Subject: [PATCH 058/118] rollback for Exception (#4069) --- .../java/com/oceanbase/odc/service/schedule/ScheduleService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 4963daf12e..883122ab44 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -214,6 +214,7 @@ public class ScheduleService { private final ScheduleMapper scheduleMapper = ScheduleMapper.INSTANCE; + @Transactional(rollbackFor = Exception.class) public List dispatchCreateSchedule(CreateFlowInstanceReq createReq) { AlterScheduleParameters parameters = (AlterScheduleParameters) createReq.getParameters(); // adapt history parameters From b688b8dab75dfff67e613ccd011cb57b9297ea10 Mon Sep 17 00:00:00 2001 From: guowl3 Date: Fri, 20 Dec 2024 17:45:58 +0800 Subject: [PATCH 059/118] disable log rolling policy (#4068) --- server/odc-server/src/main/resources/log4j2-task.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/odc-server/src/main/resources/log4j2-task.xml b/server/odc-server/src/main/resources/log4j2-task.xml index 8b084ad5e7..b79d3d78bf 100644 --- a/server/odc-server/src/main/resources/log4j2-task.xml +++ b/server/odc-server/src/main/resources/log4j2-task.xml @@ -59,11 +59,6 @@ - - - - - @@ -82,11 +77,6 @@ - - - - - From 8d902b978fc1b11004196fe66231cb9cca4debca Mon Sep 17 00:00:00 2001 From: guowl3 Date: Fri, 20 Dec 2024 18:00:47 +0800 Subject: [PATCH 060/118] fix change log detail (#4070) --- .../oceanbase/odc/service/schedule/ScheduleService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 883122ab44..2e48a6b17d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -356,10 +356,14 @@ private ScheduleChangeLog createScheduleChangelog(ScheduleChangeParams req, Sche ScheduleChangeLog changeLog = scheduleChangeLogService.createChangeLog( ScheduleChangeLog.build(targetSchedule.getId(), req.getOperationType(), - JsonUtils.toJson(targetSchedule.getParameters()), req.getOperationType() == OperationType.UPDATE - ? JsonUtils.toJson(req.getUpdateScheduleReq().getParameters()) + ? JsonUtils.toJson(targetSchedule.getParameters()) : null, + req.getOperationType() == OperationType.UPDATE + ? JsonUtils.toJson(req.getUpdateScheduleReq().getParameters()) + : req.getOperationType() == OperationType.CREATE + ? JsonUtils.toJson(req.getCreateScheduleReq().getParameters()) + : null, ScheduleChangeStatus.APPROVING)); log.info("Create change log success,changLog={}", changeLog); req.setScheduleChangeLogId(changeLog.getId()); From 6ce3415c746c822bf566af0cc74e52ff809da08e Mon Sep 17 00:00:00 2001 From: guowl3 Date: Mon, 23 Dec 2024 10:00:21 +0800 Subject: [PATCH 061/118] merge main into 4.3.x (#4079) * feat(dlm): update dlm sdk version to 1.1.4.bp1 (#3251) * fix(statefulRoute): batchCompilations and getRecyclebinSettings will failed if statefulroute enabled (#3257) * fix(resultset-export): the exported file is empty if use lower table name for oracle mode (#3254) * convert to upper case for oracle * fix typo * use lower case for mysql mode * list directory instead of looking up by name * fix(statefulRoute): sensitive column may can't get result (#3261) * fix statefulroute * fix statefulroute * fix(manual execution): manual execution button should not be displayed after clicking manual execution in odc431 (#3279) * Turn off the manual execution button display * add enum PRE_EXECUTION for FlowStatus * remove enum PRE_EXECUTION for FlowStatus * feat(dlm): support configure sharding strategy (#3275) * support configure sharding strategy. * upgrade version to 1.1.4.bp2 * fix(taskframework): cherry-pick the bug fix from obcloud_202409bp (#3909) * build: update 4.3.2 submodule (#3924) * merge 24v4bp into main (#3925) * fix(schedule): list task by creator invalid (#3752) * fix(schedule): optimize the logic for listing schedules and scheduling task (#3769) * opt list schedule & schedue task * bugfix * opt list schedule task * opt list schedule task * opt list schedule task * fix list is empty * support filter by json filed * simply code * simply code * rsp comments * rsp comments * feat(object-storage): use s3 sdk for google cloud storage (#3785) * security(object-storage): add user id into object key (#3786) * fix(dlm): do not print logs if the data source is uninitialized (#3819) * fix(schedule): customize description generator according to different environments (#3815) * Customize description generator according to different environments * code format * fix(dlm): opt error message when editing the limit configuration (#3817) * opt error message * add i18 * rsp comments * feat(schedule):support list schedule filter by datasource name (#3855) * fix(schedule): allowed to filter by datasource name #3859 * fix(schedule): list datasources with attributes (#3862) * support list schedule filter by datasource name. * list datasource with attributes * fix(schedule): schedule can be edited while subtasks are still in progress (#3891) * bugfix * bugfix * fix(database-change): delete with check schedule task status (#3899) * fix:delete schedule with check schedule task status * response comment. * code format --------- Co-authored-by: kiko Co-authored-by: LuckyLeo * fix (#3934) * fix(osc): add version whitelist to enable lock table feature * fix(session): add svrIp in session list (#3961) * fix duplicate sessionId * fix * fix(session): add non support kill version (#3979) * fix * fix * fix * fix * fix * change to data.sql * change to data.sql * change to data.sql * fix(actuator): diasble actuator by default (#3991) * default disable actuator * default disable actuator * default disable actuator * fix the issue of table not exist (#4003) * feat(common): reduce log of JsonUtils * update 432bp2 submodule (#4007) * fix(actuator): disable actuator autoconfiguration in client mode (#4047) * default disable actuator * default disable actuator * rsp comments --------- Co-authored-by: Ang <43255684+ungreat@users.noreply.github.com> Co-authored-by: LuckyLeo Co-authored-by: zijia.cj Co-authored-by: IL MARE Co-authored-by: kiko Co-authored-by: LioRoger Co-authored-by: pynzzZ --- .../tools/dbbrowser/model/DBSession.java | 5 + .../OBMySQLNoLessThan400StatsAccessor.java | 1 + .../OBOracleNoLessThan400StatsAccessor.java | 1 + pom.xml | 2 +- .../com/oceanbase/odc/ITConfigurations.java | 12 ++ .../cloud/GCSObjectStorageServiceIT.java | 36 ++++++ .../oceanbase/odc/common/json/JsonUtils.java | 8 +- .../odc/common/util/VersionUtils.java | 4 + .../odc/common/util/VersionUtilsTest.java | 7 ++ .../odc/core/shared/constant/LimitMetric.java | 4 +- .../odc/core/shared/model/OdcDBSession.java | 1 + .../i18n/BusinessMessages.properties | 3 + .../i18n/BusinessMessages_zh_CN.properties | 2 + .../i18n/BusinessMessages_zh_TW.properties | 2 + .../controller/v2/DBRecyclebinController.java | 2 +- .../web/controller/v2/PLController.java | 4 +- .../web/controller/v2/ScheduleController.java | 17 +-- .../controller/v2/ScheduleControllerHist.java | 2 +- .../config/application-clientMode.yml | 79 +++++++++++- .../src/main/resources/config/application.yml | 10 +- server/odc-server/src/main/resources/data.sql | 7 +- .../metadb/schedule/ScheduleRepository.java | 15 --- .../schedule/ScheduleTaskRepository.java | 16 +++ .../metadb/schedule/ScheduleTaskSpecs.java | 2 +- .../SensitiveColumnScanningTaskManager.java | 9 +- .../datasecurity/SensitiveColumnService.java | 15 +-- .../SensitiveColumnScanningTaskInfo.java | 6 +- .../oceanbase/odc/service/db/DBPLService.java | 7 +- .../odc/service/dlm/DLMJobStore.java | 3 + .../odc/service/dlm/DataSourceInfoMapper.java | 9 -- .../odc/service/dlm/DlmLimiterService.java | 8 +- .../feature/VersionDiffConfigService.java | 21 ++++ .../flow/util/DescriptionGenerator.java | 13 -- .../client/CloudObjectStorageClient.java | 11 ++ .../client/LocalObjectStorageClient.java | 8 ++ .../client/ObjectStorageClient.java | 2 + .../cloud/CloudObjectStorage.java | 3 + .../cloud/CloudObjectStorageService.java | 5 +- .../cloud/CloudResourceConfigurations.java | 5 + .../cloud/client/AlibabaCloudClient.java | 13 ++ .../cloud/client/AmazonCloudClient.java | 11 ++ .../cloud/client/NullCloudClient.java | 6 + .../cloud/model/DeleteObjectRequest.java | 41 +++++++ .../cloud/util/CloudObjectStorageUtil.java | 1 + .../onlineschemachange/OscService.java | 8 +- .../OnlineSchemaChangeProperties.java | 2 + .../oscfms/action/oms/OmsSwapTableAction.java | 8 +- .../rename/DefaultRenameTableInvoker.java | 11 +- .../rename/LockRenameTableFactory.java | 7 +- .../rename/LockTableSupportDecider.java | 82 +++++++++++++ .../rename/OscDBUserUtil.java | 5 +- .../OnlineSchemaChangeValidator.java | 4 +- .../odc/service/schedule/ScheduleService.java | 112 +++++++----------- .../schedule/ScheduleTaskConfiguration.java | 8 ++ .../service/schedule/ScheduleTaskService.java | 74 +----------- .../ScheduleResponseMapperFactory.java | 78 +++++++++++- .../service/schedule/job/DataDeleteJob.java | 1 + .../schedule/model/QueryScheduleParams.java | 5 +- .../model/QueryScheduleTaskParams.java | 10 +- .../model/ScheduleTaskListOverview.java | 14 +-- .../DefaultScheduleDescriptionGenerator.java | 39 ++++++ .../util/ScheduleDescriptionGenerator.java | 29 +++++ .../base/dataarchive/DataArchiveTask.java | 7 +- .../action/oms/OmsSwapTableActionTest.java | 5 +- .../rename/LockTableSupportDeciderTest.java | 90 ++++++++++++++ .../rename/OscDBUserUtilTest.java | 6 +- 66 files changed, 783 insertions(+), 261 deletions(-) create mode 100644 server/integration-test/src/test/java/com/oceanbase/odc/service/objectstorage/cloud/GCSObjectStorageServiceIT.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/model/DeleteObjectRequest.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/LockTableSupportDecider.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/util/DefaultScheduleDescriptionGenerator.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/util/ScheduleDescriptionGenerator.java create mode 100644 server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/rename/LockTableSupportDeciderTest.java diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBSession.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBSession.java index dbac2785f3..e931c023b9 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBSession.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBSession.java @@ -70,6 +70,11 @@ public class DBSession { * 客户端主机名称 */ private String host; + + /** + * 服务端地址（带端口号） + */ + private String svrIp; /** * OB Proxy HOST */ diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java index f7748864b7..a219659da4 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java @@ -41,6 +41,7 @@ public class OBMySQLNoLessThan400StatsAccessor extends OBMySQLStatsAccessor { + " STATE, " + " `USER_CLIENT_IP` as HOST, " + " HOST as PROXY_HOST, " + + " CONCAT(SVR_IP, ':', SVR_PORT) AS SVR_IP, " + " TIME as EXECUTE_TIME, " + " CASE " + " WHEN `TRANS_STATE` IS NULL OR `TRANS_STATE` IN ('', 'IDLE', 'IN_TERMINATE', 'ABORTED', " diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java index 436652f3c2..cb727753a4 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java @@ -38,6 +38,7 @@ public class OBOracleNoLessThan400StatsAccessor extends BaseOBOracleStatsAccesso + " STATE, " + " USER_CLIENT_IP as HOST, " + " HOST as PROXY_HOST, " + + " SVR_IP || ':' || TO_CHAR(SVR_PORT) AS SVR_IP, " + " TIME as EXECUTE_TIME, " + " CASE " + " WHEN TRANS_STATE IS NULL OR TRANS_STATE IN ('', 'IDLE', 'IN_TERMINATE', 'ABORTED', " diff --git a/pom.xml b/pom.xml index ed7c8770f6..aa9c591c75 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ 2.1.6 - 1.1.6.bp + 1.1.6.bp2 2.11.0 diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/ITConfigurations.java b/server/integration-test/src/test/java/com/oceanbase/odc/ITConfigurations.java index 458e15d5b9..2aedae416e 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/ITConfigurations.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/ITConfigurations.java @@ -86,6 +86,18 @@ public static ObjectStorageConfiguration getOBSConfiguration() { return configuration; } + public static ObjectStorageConfiguration getGCSConfiguration() { + ObjectStorageConfiguration configuration = new ObjectStorageConfiguration(); + configuration.setCloudProvider(CloudProvider.GOOGLE_CLOUD); + configuration.setPublicEndpoint(get("odc.cloud.object-storage.gcs.endpoint")); + configuration.setAccessKeyId(get("odc.cloud.object-storage.gcs.access-key-id")); + configuration.setAccessKeySecret(get("odc.cloud.object-storage.gcs.access-key-secret")); + configuration.setBucketName(get("odc.cloud.object-storage.gcs.bucket-name")); + configuration.setRoleArn(get("odc.cloud.object-storage.gcs.role-arn")); + configuration.setRoleSessionName(get("odc.cloud.object-storage.gcs.role-session-name")); + return configuration; + } + public static OnlineSchemaChangeProperties getOscPrivateCloudProperties() { OnlineSchemaChangeProperties configuration = new OnlineSchemaChangeProperties(); OmsProperties omsConfiguration = new OmsProperties(); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/objectstorage/cloud/GCSObjectStorageServiceIT.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/objectstorage/cloud/GCSObjectStorageServiceIT.java new file mode 100644 index 0000000000..7c4d80fd04 --- /dev/null +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/objectstorage/cloud/GCSObjectStorageServiceIT.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.objectstorage.cloud; + +import org.junit.Ignore; + +import com.oceanbase.odc.ITConfigurations; +import com.oceanbase.odc.service.objectstorage.cloud.client.CloudClient; +import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; + +@Ignore +public class GCSObjectStorageServiceIT extends AbstractCloudObjectStorageServiceTest { + + @Override + CloudObjectStorageService createCloudObjectStorageService() { + ObjectStorageConfiguration configuration = ITConfigurations.getGCSConfiguration(); + CloudClient cloudClient = new CloudResourceConfigurations().publicEndpointCloudClient(() -> configuration); + CloudClient internalCloudClient = + new CloudResourceConfigurations().internalEndpointCloudClient(() -> configuration); + return new CloudObjectStorageService(cloudClient, internalCloudClient, () -> configuration); + } + +} diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/json/JsonUtils.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/json/JsonUtils.java index b26258bd3e..669d874f81 100644 --- a/server/odc-common/src/main/java/com/oceanbase/odc/common/json/JsonUtils.java +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/json/JsonUtils.java @@ -72,7 +72,7 @@ public static T fromJson(String json, Class classType) { try { return OBJECT_MAPPER.readValue(json, classType); } catch (JsonProcessingException e) { - log.warn("deserialize str = {} failed", json, e); + log.warn("deserialize str = {} failed, reason = {}", json, e.getMessage()); return null; } } @@ -95,7 +95,7 @@ public static T fromJson(String json, JavaType javaType) { try { return OBJECT_MAPPER.readValue(json, javaType); } catch (JsonProcessingException e) { - log.warn("deserialize str = {} failed", json, e); + log.warn("deserialize str = {} failed, reason = {}", json, e.getMessage()); return null; } } @@ -111,14 +111,14 @@ public static T fromJson(String json, TypeReference valueTypeRef) { try { return OBJECT_MAPPER.readValue(json, valueTypeRef); } catch (JsonProcessingException e) { - log.warn("deserialize str = {} failed", json, e); + log.warn("deserialize str = {} failed, reason = {}", json, e.getMessage()); return null; } } /** * Ignore deserialize fail when json missing property annotation by @JsonTypeInfo - * + * * @param json json string * @param valueTypeRef represent the actual type of generic * @return object represented by valueTypeRef diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/VersionUtils.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/VersionUtils.java index 394605e99c..aadb435f13 100644 --- a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/VersionUtils.java +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/VersionUtils.java @@ -34,6 +34,10 @@ public static boolean isGreaterThan(String currentVersion, String targetVersion) return compareVersions(currentVersion, targetVersion) > 0; } + public static boolean isGreaterThan0(String currentVersion) { + return compareVersions(currentVersion, "0") > 0; + } + public static boolean isLessThanOrEqualsTo(String currentVersion, String targetVersion) { return !isGreaterThan(currentVersion, targetVersion); } diff --git a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/VersionUtilsTest.java b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/VersionUtilsTest.java index af436c263c..50290b78f6 100644 --- a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/VersionUtilsTest.java +++ b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/VersionUtilsTest.java @@ -50,6 +50,13 @@ public void isGreaterThan_Length4() { Assert.assertFalse(VersionUtils.isGreaterThan("1.1.0.0", "1.1.1.0")); } + @Test + public void isGreaterThan0_Length() { + Assert.assertTrue(VersionUtils.isGreaterThan0("1.1.2")); + Assert.assertFalse(VersionUtils.isGreaterThan0("-1")); + Assert.assertFalse(VersionUtils.isGreaterThan0("0")); + } + @Test public void isGreaterThan_Length4WithLength3() { Assert.assertTrue(VersionUtils.isGreaterThan("1.1.1.1", "1.1.1")); diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/LimitMetric.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/LimitMetric.java index 29b531431b..cde4c207f8 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/LimitMetric.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/LimitMetric.java @@ -47,7 +47,9 @@ public enum LimitMetric implements Translatable { WORKSHEET_CHANGE_COUNT, WORKSHEET_SAME_LEVEL_COUNT, WORKSHEET_COUNT_IN_PROJECT, - WORKSPACE_COUNT_IN_PROJECT; + WORKSPACE_COUNT_IN_PROJECT, + DLM_ROW_LIMIT, + DLM_DATA_SIZE_LIMIT; @Override public String code() { diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/OdcDBSession.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/OdcDBSession.java index 13113e18a9..461421df5c 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/OdcDBSession.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/model/OdcDBSession.java @@ -45,6 +45,7 @@ public static OdcDBSession from(DBSession dbSession) { session.setStatus(dbSession.getState()); session.setObproxyIp(dbSession.getProxyHost()); session.setSql(dbSession.getLatestQueries()); + session.setSvrIp(dbSession.getSvrIp()); return session; } } diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties index aa1a7cdd05..ccc8a223ae 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties @@ -142,6 +142,9 @@ com.oceanbase.odc.LimitMetric.WORKSHEET_CHANGE_COUNT=The number of changed works com.oceanbase.odc.LimitMetric.WORKSHEET_SAME_LEVEL_COUNT=The number of worksheets at the same level com.oceanbase.odc.LimitMetric.WORKSHEET_COUNT_IN_PROJECT=The number of worksheets in the project com.oceanbase.odc.LimitMetric.WORKSPACE_COUNT_IN_PROJECT=The number of workspaces in the project +com.oceanbase.odc.LimitMetric.DLM_ROW_LIMIT=Row limit +com.oceanbase.odc.LimitMetric.DLM_DATA_SIZE_LIMIT=Data size limit + # # ConnectionAccountType # diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties index a0f21e1986..0dc888df39 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties @@ -142,6 +142,8 @@ com.oceanbase.odc.LimitMetric.WORKSHEET_CHANGE_COUNT=变更的工作簿数量 com.oceanbase.odc.LimitMetric.WORKSHEET_SAME_LEVEL_COUNT=同级的工作簿数量 com.oceanbase.odc.LimitMetric.WORKSHEET_COUNT_IN_PROJECT=项目的工作簿数量 com.oceanbase.odc.LimitMetric.WORKSPACE_COUNT_IN_PROJECT=项目的工作空间数量 +com.oceanbase.odc.LimitMetric.DLM_ROW_LIMIT=行限流 +com.oceanbase.odc.LimitMetric.DLM_DATA_SIZE_LIMIT=数据大小限流 # # ConnectionAccountType # diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties index a10689377e..6d3b3ae0a9 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties @@ -142,6 +142,8 @@ com.oceanbase.odc.LimitMetric.WORKSHEET_CHANGE_COUNT=变更的工作簿数量 com.oceanbase.odc.LimitMetric.WORKSHEET_SAME_LEVEL_COUNT=同級的工作簿數量 com.oceanbase.odc.LimitMetric.WORKSHEET_COUNT_IN_PROJECT=項目的工作簿數量 com.oceanbase.odc.LimitMetric.WORKSPACE_COUNT_IN_PROJECT=項目的工作空間數量 +com.oceanbase.odc.LimitMetric.DLM_ROW_LIMIT=行限流 +com.oceanbase.odc.LimitMetric.DLM_DATA_SIZE_LIMIT=數據大小限流 # # ConnectionAccountType diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBRecyclebinController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBRecyclebinController.java index 7be5c77218..0985fb112a 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBRecyclebinController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBRecyclebinController.java @@ -103,7 +103,7 @@ public SuccessResponse getRecyclebinSettings(@PathVariable S @ApiOperation(value = "updateRecyclebinSettings", notes = "更新回收站设置，sid示例：sid:1000-1:d:db1") @RequestMapping(value = "/settings", method = RequestMethod.PATCH) - @StatefulRoute(multiState = true, stateIdExpression = "#req.sessionIds", + @StatefulRoute(stateIdExpression = "#req.sessionIds", stateManager = "dbRecyclebinUpdateStateManager") public SuccessResponse updateRecyclebinSettings(@RequestBody UpdateRecyclebinSettingsReq req) { List sessions = req.getSessionIds().stream().map(s -> { diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/PLController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/PLController.java index c02c6e25c9..f41216511c 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/PLController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/PLController.java @@ -61,7 +61,7 @@ public SuccessResponse startBatchCompile(@PathVariable String sessionId, @ApiOperation(value = "end batchCompile", notes = "终止批量编译") @RequestMapping(value = {"/{sessionId}/databases/{databaseName}/batchCompilations/{id}", "/{sessionId}/currentDatabase/batchCompilations/{id}"}, method = RequestMethod.DELETE) - @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sessionId") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse endBatchCompile(@PathVariable String id) { return Responses.success(plService.endBatchCompile(id)); } @@ -69,7 +69,7 @@ public SuccessResponse endBatchCompile(@PathVariable String id) { @ApiOperation(value = "get result for batchCompile", notes = "获取批量编译PL对象的结果") @RequestMapping(value = {"/{sessionId}/databases/{databaseName}/batchCompilations/{id}", "/{sessionId}/currentDatabase/batchCompilations/{id}"}, method = RequestMethod.GET) - @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#id") + @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#id") public SuccessResponse getBatchCompileResult(@PathVariable String id) { return Responses.success(plService.getBatchCompileResult(id)); } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleController.java index ce621d299a..b7112d7996 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleController.java @@ -56,7 +56,6 @@ import com.oceanbase.odc.service.schedule.model.ScheduleTaskListOverview; import com.oceanbase.odc.service.schedule.model.ScheduleTaskOverview; import com.oceanbase.odc.service.schedule.model.ScheduleType; -import com.oceanbase.odc.service.schedule.model.TriggerStrategy; import com.oceanbase.odc.service.schedule.model.UpdateScheduleReq; import com.oceanbase.odc.service.task.executor.logger.LogUtils; import com.oceanbase.odc.service.task.model.OdcTaskLogLevel; @@ -153,7 +152,7 @@ public SuccessResponse detailScheduleTask(@PathVariable } @RequestMapping(value = "/schedules/{scheduleId:[\\d]+}/tasks", method = RequestMethod.GET) - public PaginatedResponse listTask( + public PaginatedResponse listScheduleTaskForSchedule( @PageableDefault(size = Integer.MAX_VALUE, sort = {"id"}, direction = Direction.DESC) Pageable pageable, @PathVariable Long scheduleId) { return Responses.paginated(scheduleService.listScheduleTaskOverview(pageable, scheduleId)); @@ -161,14 +160,14 @@ public PaginatedResponse listTask( @RequestMapping(value = "/tasks", method = RequestMethod.GET) - public PaginatedResponse listAllTask( + public PaginatedResponse listScheduleTask( @PageableDefault(size = Integer.MAX_VALUE, sort = {"id"}, direction = Direction.DESC) Pageable pageable, @RequestParam(required = false, name = "dataSourceId") Set datasourceIds, @RequestParam(required = false, name = "databaseName") String databaseName, @RequestParam(required = false, name = "tenantId") String tenantId, @RequestParam(required = false, name = "clusterId") String clusterId, - @RequestParam(required = false, name = "id") Long id, - @RequestParam(required = false, name = "scheduleId") Long scheduleId, + @RequestParam(required = false, name = "id") String id, + @RequestParam(required = false, name = "scheduleId") String scheduleId, @RequestParam(required = false, name = "scheduleName") String scheduleName, @RequestParam(required = false, name = "status") List status, @RequestParam(required = true, name = "scheduleType") ScheduleType scheduleType, @@ -193,7 +192,7 @@ public PaginatedResponse listAllTask( .projectId(projectId) .build(); - return Responses.paginated(scheduleService.listScheduleTaskOverviewByScheduleType(pageable, req)); + return Responses.paginated(scheduleService.listScheduleTaskListOverview(pageable, req)); } // schedule @@ -238,10 +237,11 @@ public SuccessResponse createSchedule(@RequestBody CreateScheduleReq r public PaginatedResponse list( @PageableDefault(size = Integer.MAX_VALUE, sort = {"id"}, direction = Direction.DESC) Pageable pageable, @RequestParam(required = false, name = "dataSourceId") Set datasourceIds, + @RequestParam(required = false, name = "dataSourceName") String dataSourceName, @RequestParam(required = false, name = "databaseName") String databaseName, @RequestParam(required = false, name = "tenantId") String tenantId, @RequestParam(required = false, name = "clusterId") String clusterId, - @RequestParam(required = false, name = "id") Long id, + @RequestParam(required = false, name = "id") String id, @RequestParam(required = false, name = "name") String name, @RequestParam(required = false, name = "status") List status, @RequestParam(required = false, name = "type") ScheduleType type, @@ -250,10 +250,11 @@ public PaginatedResponse list( @RequestParam(required = false, name = "creator") String creator, @RequestParam(required = false, name = "projectUniqueIdentifier") String projectUniqueIdentifier, @RequestParam(required = false, name = "projectId") Long projectId, - @RequestParam(required = false, name = "triggerStrategy") TriggerStrategy triggerStrategy) { + @RequestParam(required = false, name = "triggerStrategy") String triggerStrategy) { QueryScheduleParams req = QueryScheduleParams.builder() .id(id) .name(name) + .dataSourceName(dataSourceName) .dataSourceIds(datasourceIds) .databaseName(databaseName) .tenantId(tenantId) diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java index a757ec92de..bcee282d1e 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java @@ -65,7 +65,7 @@ public class ScheduleControllerHist { public PaginatedResponse list( @PageableDefault(size = Integer.MAX_VALUE, sort = {"id"}, direction = Direction.DESC) Pageable pageable, @RequestParam(required = false, name = "connectionId") Set connectionIds, - @RequestParam(required = false, name = "id") Long id, + @RequestParam(required = false, name = "id") String id, @RequestParam(required = false, name = "status") List status, @RequestParam(required = false, name = "type") ScheduleType type, @RequestParam(required = false, name = "startTime") Date startTime, diff --git a/server/odc-server/src/main/resources/config/application-clientMode.yml b/server/odc-server/src/main/resources/config/application-clientMode.yml index 3117464042..6ee099ef2a 100644 --- a/server/odc-server/src/main/resources/config/application-clientMode.yml +++ b/server/odc-server/src/main/resources/config/application-clientMode.yml @@ -21,5 +21,82 @@ spring: - org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration - org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration - org.springframework.boot.autoconfigure.security.oauth2.client.servlet.OAuth2ClientAutoConfiguration + - org.springframework.boot.actuate.autoconfigure.metrics.export.prometheus.PrometheusMetricsExportAutoConfiguration jpa: - database-platform: org.hibernate.dialect.H2Dialect \ No newline at end of file + database-platform: org.hibernate.dialect.H2Dialect +management: + endpoints: + enabled-by-default: false + web: + exposure: + include: [] + metrics: + export: + simple: + enabled: false + appoptics: + enabled: false + atlas: + enabled: false + datadog: + enabled: false + dynatrace: + enabled: false + elastic: + enabled: false + ganglia: + enabled: false + graphite: + enabled: false + influx: + enabled: false + jmx: + enabled: false + kairos: + enabled: false + newrelic: + enabled: false + prometheus: + enabled: false + signalfx: + enabled: false + stackdriver: + enabled: false + statsd: + enabled: false + wavefront: + enabled: false + health: + defaults: + enabled: false + diskspace: + enabled: false + db: + enabled: false + auditevents: + enabled: false + endpoint: + loggers: + enabled: false + threaddump: + enabled: false + env: + enabled: false + beans: + enabled: false + conditions: + enabled: false + configprops: + enabled: false + info: + enabled: false + httptrace: + enabled: false + mappings: + enabled: false + scheduledtasks: + enabled: false + sessions: + enabled: false + shutdown: + enabled: false \ No newline at end of file diff --git a/server/odc-server/src/main/resources/config/application.yml b/server/odc-server/src/main/resources/config/application.yml index 63148ec438..7afa094517 100644 --- a/server/odc-server/src/main/resources/config/application.yml +++ b/server/odc-server/src/main/resources/config/application.yml @@ -79,8 +79,12 @@ management: web: exposure: include: "business,application" + enabled-by-default: ${odc.system.monitor.actuator.enabled:false} + metrics: + export: + prometheus: + enabled: ${odc.system.monitor.actuator.enabled:false} + endpoint: prometheus: - enabled: true - server: - port: 8089 + enabled: ${odc.system.monitor.actuator.enabled:false} diff --git a/server/odc-server/src/main/resources/data.sql b/server/odc-server/src/main/resources/data.sql index 010deb027f..3e9ce4e440 100644 --- a/server/odc-server/src/main/resources/data.sql +++ b/server/odc-server/src/main/resources/data.sql @@ -854,4 +854,9 @@ INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('o 'false', 'Enable secure cookie or not, default value false') ON DUPLICATE KEY UPDATE `id`=`id`; INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('odc.features.logicaldatabase.enabled', 'true', - 'Whether to enable the logical database feature, default is true, indicating enabled.') ON DUPLICATE KEY UPDATE `id`=`id`; +'Whether to enable the logical database feature, default is true, indicating enabled.') ON DUPLICATE KEY UPDATE `id`=`id`; + +INSERT INTO `config_system_configuration` (`key`, `value`, `application`, `profile`, `label`, `description`) +VALUES ('odc.session.kill-query-or-session.max-supported-ob-version', '4.2.5', 'odc', 'default', 'master', + 'Max OBVersion kill session or kill query supported, only take effect when value greater than 0') +ON DUPLICATE KEY UPDATE `id`=`id`; \ No newline at end of file diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java index 469844b456..ee8dde0c7b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java @@ -76,19 +76,4 @@ default Page find(@NotNull Pageable pageable, @NotNull QuerySche .and(OdcJpaRepository.eq(ScheduleEntity_.organizationId, params.getOrganizationId())); return findAll(specification, pageable); } - - default List find(@NotNull QueryScheduleParams params) { - Specification specification = Specification - .where(OdcJpaRepository.between(ScheduleEntity_.createTime, params.getStartTime(), params.getEndTime())) - .and(OdcJpaRepository.in(ScheduleEntity_.dataSourceId, params.getDataSourceIds())) - .and(OdcJpaRepository.eq(ScheduleEntity_.databaseName, params.getDatabaseName())) - .and(OdcJpaRepository.eq(ScheduleEntity_.type, params.getType())) - .and(OdcJpaRepository.in(ScheduleEntity_.projectId, params.getProjectIds())) - .and(OdcJpaRepository.eq(ScheduleEntity_.id, params.getId())) - .and(OdcJpaRepository.notEq(ScheduleEntity_.status, ScheduleStatus.DELETED)) - .and(OdcJpaRepository.in(ScheduleEntity_.creatorId, params.getCreatorIds())) - .and(OdcJpaRepository.like(ScheduleEntity_.name, params.getName())) - .and(OdcJpaRepository.eq(ScheduleEntity_.organizationId, params.getOrganizationId())); - return findAll(specification); - } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepository.java index 8a62b9d83c..bcd9639d03 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepository.java @@ -20,7 +20,11 @@ import java.util.Set; import javax.transaction.Transactional; +import javax.validation.constraints.NotNull; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Modifying; @@ -28,7 +32,9 @@ import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; +import com.oceanbase.odc.config.jpa.OdcJpaRepository; import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.schedule.model.QueryScheduleTaskParams; /** * @Author：tinker @@ -100,4 +106,14 @@ int updateStatusById(@Param("id") Long id, @Param("newStatus") TaskStatus newSta + "st.resultJson=:#{#entity.resultJson} where st.id=:#{#entity.id}") int update(@Param("entity") ScheduleTaskEntity entity); + default Page find(@NotNull Pageable pageable, @NotNull QueryScheduleTaskParams params) { + Specification specification = Specification + .where(OdcJpaRepository.between(ScheduleTaskEntity_.createTime, params.getStartTime(), + params.getEndTime())) + .and(OdcJpaRepository.eq(ScheduleTaskEntity_.id, params.getId())) + .and(OdcJpaRepository.in(ScheduleTaskEntity_.status, params.getStatuses())) + .and(OdcJpaRepository.in(ScheduleTaskEntity_.jobName, params.getScheduleIds())); + return findAll(specification, pageable); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskSpecs.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskSpecs.java index 4d28fd5a37..a207ae63d4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskSpecs.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskSpecs.java @@ -43,7 +43,7 @@ public static Specification jobNameIn(Set jobName) { return SpecificationUtil.columnIn("jobName", jobName); } - public static Specification jobIdEquals(Long id) { + public static Specification idEquals(Long id) { return SpecificationUtil.columnEqual("id", id); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnScanningTaskManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnScanningTaskManager.java index fb5b260b4f..72a93c1676 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnScanningTaskManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnScanningTaskManager.java @@ -43,6 +43,7 @@ import com.oceanbase.odc.service.datasecurity.model.SensitiveRule; import com.oceanbase.odc.service.db.browser.DBSchemaAccessors; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; +import com.oceanbase.odc.service.state.StatefulUuidStateIdGenerator; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor; @@ -56,11 +57,12 @@ @Component public class SensitiveColumnScanningTaskManager { + private final SensitiveColumnScanningResultCache cache = SensitiveColumnScanningResultCache.getInstance(); @Autowired @Qualifier("scanSensitiveColumnExecutor") private ThreadPoolTaskExecutor executor; - - private final SensitiveColumnScanningResultCache cache = SensitiveColumnScanningResultCache.getInstance(); + @Autowired + private StatefulUuidStateIdGenerator statefulUuidStateIdGenerator; public SensitiveColumnScanningTaskInfo start(List databases, List rules, ConnectionConfig connectionConfig, Map> databaseId2SensitiveColumns) { @@ -84,7 +86,8 @@ public SensitiveColumnScanningTaskInfo start(List databases, List>> runningTaskMap; @@ -181,7 +184,7 @@ public String startBatchCompile(@NonNull ConnectionSession session, if (StringUtils.isBlank(databaseName)) { // it means batch compile refers to current database handle = taskManager.submit(taskCallable); - taskId = UUID.randomUUID().toString(); + taskId = statefulUuidStateIdGenerator.generateStateId("BatchCompile"); } else { throw new UnsupportedException("Batch compile PL in another database is not supported yet"); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index 94414bf025..ba5d1a4be9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -71,6 +71,9 @@ public void setDlmTableUnits(Map dlmTableUnits) { } public void destroy() { + if (dataSource == null) { + return; + } try { dataSource.close(); } catch (Exception e) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java index f817395291..56e8a383be 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java @@ -28,7 +28,6 @@ import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; import com.oceanbase.tools.migrator.common.configure.DataSourceInfo; import com.oceanbase.tools.migrator.common.enums.DataBaseType; -import com.oceanbase.tools.migrator.common.util.EncryptUtils; import lombok.extern.slf4j.Slf4j; @@ -87,15 +86,7 @@ public static DataSourceInfo toDataSourceInfo(ConnectionConfig connectionConfig, dataSourceInfo .setFullUserName(OBConsoleDataSourceFactory.getUsername(connectionConfig)); dataSourceInfo.setDatabaseType(DataBaseType.OB_MYSQL); - dataSourceInfo.setSysUser(connectionConfig.getSysTenantUsername()); dataSourceInfo.setClusterName(connectionConfig.getClusterName()); - if (StringUtils.isNotEmpty(connectionConfig.getSysTenantPassword())) { - try { - dataSourceInfo.setSysPassword(EncryptUtils.encode(connectionConfig.getSysTenantPassword())); - } catch (Exception e) { - throw new RuntimeException(e); - } - } dataSourceInfo.setSysDatabaseName("oceanbase"); break; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DlmLimiterService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DlmLimiterService.java index fef8114b5c..373ca1f494 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DlmLimiterService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DlmLimiterService.java @@ -26,6 +26,8 @@ import org.springframework.transaction.annotation.Transactional; import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.LimitMetric; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.metadb.dlm.DlmLimiterConfigEntity; @@ -119,10 +121,12 @@ public RateLimitConfiguration getDefaultLimiterConfig() { private void checkLimiterConfig(RateLimitConfiguration limiterConfig) { if (limiterConfig.getRowLimit() != null && limiterConfig.getRowLimit() > maxRowLimit) { - throw new IllegalArgumentException(String.format("The maximum row limit is %s rows/s.", maxRowLimit)); + PreConditions.lessThanOrEqualTo("rowLimit", LimitMetric.DLM_ROW_LIMIT, limiterConfig.getRowLimit(), + maxRowLimit); } if (limiterConfig.getDataSizeLimit() != null && limiterConfig.getDataSizeLimit() > maxDataSizeLimit) { - throw new IllegalArgumentException(String.format("The maximum data size is %s KB/s.", maxDataSizeLimit)); + PreConditions.lessThanOrEqualTo("dataSizeLimit", LimitMetric.DLM_DATA_SIZE_LIMIT, + limiterConfig.getDataSizeLimit(), maxDataSizeLimit); } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java index c94bf8a35c..ee5d4f2cf6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java @@ -17,6 +17,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -46,8 +47,12 @@ @SkipAuthorize("inside connect session") public class VersionDiffConfigService { private static final String SUPPORT_PREFIX = "support"; + private static final String MAX_SUPPORT_KILL_OB_VERSION = + "odc.session.kill-query-or-session.max-supported-ob-version"; private static final String SUPPORT_PROCEDURE = "support_procedure"; private static final String SUPPORT_FUNCTION = "support_function"; + private static final String SUPPORT_KILL_SESSION = "support_kill_session"; + private static final String SUPPORT_KILL_QUERY = "support_kill_query"; private static final String SUPPORT_PL_DEBUG = "support_pl_debug"; private static final String SUPPORT_EXTERNAL_TABLE = "support_external_table"; private static final String COLUMN_DATA_TYPE = "column_data_type"; @@ -129,6 +134,22 @@ public List getSupportFeatures(ConnectionSession connectionSession) { obSupport.setSupport(false); } } + + // killSession that is greater than the specified version is currently not supported + if (SUPPORT_KILL_SESSION.equalsIgnoreCase(configKey) + || SUPPORT_KILL_QUERY.equalsIgnoreCase(configKey)) { + Optional nonSupport = systemConfigs.stream().filter( + c -> c.getKey().equalsIgnoreCase(MAX_SUPPORT_KILL_OB_VERSION)).findFirst(); + if (nonSupport.isPresent()) { + String maxSupportVersion = nonSupport.get().getValue(); + // maxSupportVersion takes effect only greater than 0 + if (VersionUtils.isGreaterThan0(maxSupportVersion) + && VersionUtils.isGreaterThanOrEqualsTo(currentVersion, maxSupportVersion)) { + obSupport.setSupport(false); + } + } + } + if (SUPPORT_PL_DEBUG.equalsIgnoreCase(configKey) && !isPLDebugSupport(connectionSession)) { obSupport.setSupport(false); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/DescriptionGenerator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/DescriptionGenerator.java index d5e7a01177..5b8dcbae67 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/DescriptionGenerator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/util/DescriptionGenerator.java @@ -23,7 +23,6 @@ import com.oceanbase.odc.service.databasechange.model.DatabaseChangeDatabase; import com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq; import com.oceanbase.odc.service.flow.task.model.MultipleDatabaseChangeParameters; -import com.oceanbase.odc.service.schedule.model.ScheduleChangeParams; /** * @Author：tinker @@ -51,16 +50,4 @@ public static void generateDescription(CreateFlowInstanceReq req) { } } - public static void generateScheduleDescription(ScheduleChangeParams req) { - if (StringUtils.isEmpty(req.getCreateScheduleReq().getDescription())) { - String environmentName = req.getEnvironmentName(); - String connectionName = req.getConnectionName(); - String databaseName = req.getDatabaseName(); - String description = - StringUtils.isEmpty(connectionName) ? String.format("[%s]%s", environmentName, databaseName) - : String.format("[%s]%s.%s", environmentName, connectionName, databaseName); - req.getCreateScheduleReq().setDescription(description); - } - } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java index 19417b462b..02898febbf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java @@ -41,6 +41,7 @@ import com.oceanbase.odc.service.objectstorage.cloud.model.CloudObjectStorageConstants; import com.oceanbase.odc.service.objectstorage.cloud.model.CompleteMultipartUploadRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.CompleteMultipartUploadResult; +import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsResult; import com.oceanbase.odc.service.objectstorage.cloud.model.GetObjectRequest; @@ -166,6 +167,16 @@ public List deleteObjects(List objectNames) { return deletedObjects; } + @Override + public String deleteObject(String objectName) { + verifySupported(); + DeleteObjectRequest request = new DeleteObjectRequest(getBucketName(), objectName); + String deleted = internalEndpointCloudObjectStorage.deleteObject(request); + log.info("Delete file success, tryDeleteObjectName={}, deletedObjectName={}", + objectName, deleted); + return deleted; + } + @Override public InputStream getObject(String objectName) throws IOException { verifySupported(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/LocalObjectStorageClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/LocalObjectStorageClient.java index a59f4e5079..7f68929352 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/LocalObjectStorageClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/LocalObjectStorageClient.java @@ -25,6 +25,8 @@ import java.util.HashSet; import java.util.List; +import javax.validation.constraints.NotBlank; + import org.apache.commons.collections4.CollectionUtils; import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectTagging; @@ -96,6 +98,12 @@ public List deleteObjects(List objectNames) throws IOException { return new ArrayList<>(); } + @Override + public String deleteObject(@NotBlank String objectName) throws IOException { + blockOperator.deleteByObjectId(objectName); + return objectName; + } + @Override public InputStream getObject(String objectName) throws IOException { throw new UnsupportedOperationException("Not supported yet."); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/ObjectStorageClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/ObjectStorageClient.java index c383c232d9..770166abdd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/ObjectStorageClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/ObjectStorageClient.java @@ -49,6 +49,8 @@ public interface ObjectStorageClient { List deleteObjects(@NotEmpty List objectNames) throws IOException; + String deleteObject(@NotBlank String objectName) throws IOException; + InputStream getObject(@NotBlank String objectName) throws IOException; InputStream getAbortableObject(@NotBlank String objectName) throws IOException; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java index 3b4606b011..60d75792f4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java @@ -24,6 +24,7 @@ import com.oceanbase.odc.service.objectstorage.cloud.model.CompleteMultipartUploadRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.CompleteMultipartUploadResult; import com.oceanbase.odc.service.objectstorage.cloud.model.CopyObjectResult; +import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsResult; import com.oceanbase.odc.service.objectstorage.cloud.model.GetObjectRequest; @@ -69,6 +70,8 @@ CopyObjectResult copyObject(String bucketName, String from, String to) DeleteObjectsResult deleteObjects(DeleteObjectsRequest request) throws CloudException; + String deleteObject(DeleteObjectRequest request) throws CloudException; + boolean doesObjectExist(String bucketName, String key) throws CloudException; StorageObject getObject(String bucketName, String key) throws CloudException; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorageService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorageService.java index b0e7b18c46..f302713787 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorageService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorageService.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; -import java.util.Collections; import java.util.List; import java.util.UUID; @@ -164,8 +163,8 @@ public byte[] readContent(@NotBlank String objectName) throws IOException { * @throws IOException */ public boolean delete(@NotBlank String objectName) throws IOException { - List deletedObjectNames = delete(Collections.singletonList(objectName)); - return !deletedObjectNames.isEmpty(); + cloudObjectStorageClient.deleteObject(objectName); + return true; } /** diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudResourceConfigurations.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudResourceConfigurations.java index d988a0f9de..369d5a76d1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudResourceConfigurations.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudResourceConfigurations.java @@ -105,6 +105,7 @@ public CloudClient generateCloudClient(ObjectStorageConfiguration configuration, case AWS: case TENCENT_CLOUD: case HUAWEI_CLOUD: + case GOOGLE_CLOUD: return createAmazonCloudClient(configuration); default: return new NullCloudClient(); @@ -140,6 +141,10 @@ static AmazonCloudClient createAmazonCloudClient(ObjectStorageConfiguration conf .withCredentials(credentialsProvider) .withClientConfiguration(clientConfiguration) .disableChunkedEncoding(); + // GCS does not support region + if (configuration.getCloudProvider() == CloudProvider.GOOGLE_CLOUD) { + region = "EMPTY"; + } // if not AWS, means use S3 SDK to access other cloud storage, then we must set endpoint if (!configuration.getCloudProvider().isAWS()) { String endpoint = configuration.getPublicEndpoint(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java index 3a67bcb876..059a7ba14e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java @@ -20,6 +20,7 @@ import java.io.InputStream; import java.net.URL; import java.text.ParseException; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; @@ -50,6 +51,7 @@ import com.oceanbase.odc.service.objectstorage.cloud.model.CompleteMultipartUploadRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.CompleteMultipartUploadResult; import com.oceanbase.odc.service.objectstorage.cloud.model.CopyObjectResult; +import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsResult; import com.oceanbase.odc.service.objectstorage.cloud.model.GetObjectRequest; @@ -198,6 +200,17 @@ public DeleteObjectsResult deleteObjects(DeleteObjectsRequest request) throws Cl }); } + @Override + public String deleteObject(DeleteObjectRequest request) throws CloudException { + com.aliyun.oss.model.DeleteObjectsRequest ossRequest = + new com.aliyun.oss.model.DeleteObjectsRequest(request.getBucketName()) + .withKeys(Collections.singletonList(request.getKey())); + return callOssMethod("Delete object", () -> { + com.aliyun.oss.model.DeleteObjectsResult ossResult = oss.deleteObjects(ossRequest); + return ossResult.getDeletedObjects().get(0); + }); + } + @Override public boolean doesObjectExist(String bucketName, String key) throws CloudException { return callOssMethod("Check object exist", () -> oss.doesObjectExist(bucketName, key)); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java index b786892028..e8e80d226b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java @@ -51,6 +51,7 @@ import com.oceanbase.odc.service.objectstorage.cloud.model.CompleteMultipartUploadRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.CompleteMultipartUploadResult; import com.oceanbase.odc.service.objectstorage.cloud.model.CopyObjectResult; +import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsResult; import com.oceanbase.odc.service.objectstorage.cloud.model.GetObjectRequest; @@ -211,6 +212,16 @@ public DeleteObjectsResult deleteObjects(DeleteObjectsRequest request) throws Cl }); } + @Override + public String deleteObject(DeleteObjectRequest request) throws CloudException { + com.amazonaws.services.s3.model.DeleteObjectRequest s3Request = + new com.amazonaws.services.s3.model.DeleteObjectRequest(request.getBucketName(), request.getKey()); + return callAmazonMethod("Delete object", () -> { + s3.deleteObject(s3Request); + return s3Request.getKey(); + }); + } + @Override public boolean doesObjectExist(String bucketName, String key) throws CloudException { return callAmazonMethod("Check object exist", () -> s3.doesObjectExist(bucketName, key)); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java index e52c38038c..7accc9b39b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java @@ -24,6 +24,7 @@ import com.oceanbase.odc.service.objectstorage.cloud.model.CompleteMultipartUploadRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.CompleteMultipartUploadResult; import com.oceanbase.odc.service.objectstorage.cloud.model.CopyObjectResult; +import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsRequest; import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsResult; import com.oceanbase.odc.service.objectstorage.cloud.model.GetObjectRequest; @@ -87,6 +88,11 @@ public DeleteObjectsResult deleteObjects(DeleteObjectsRequest request) throws Cl throw new UnsupportedException(); } + @Override + public String deleteObject(DeleteObjectRequest request) throws CloudException { + throw new UnsupportedException(); + } + @Override public boolean doesObjectExist(String bucketName, String key) throws CloudException { return false; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/model/DeleteObjectRequest.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/model/DeleteObjectRequest.java new file mode 100644 index 0000000000..d302d2180c --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/model/DeleteObjectRequest.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.objectstorage.cloud.model; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author: liuyizhuo.lyz + * @date: 2024/10/29 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class DeleteObjectRequest extends GenericRequest { + public DeleteObjectRequest() {} + + public DeleteObjectRequest(String bucketName) { + super(bucketName); + } + + public DeleteObjectRequest(String bucketName, String key) { + super(bucketName, key); + } + + public DeleteObjectRequest(String bucketName, String key, String versionId) { + super(bucketName, key, versionId); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/util/CloudObjectStorageUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/util/CloudObjectStorageUtil.java index ae86a6b798..7d495b84fa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/util/CloudObjectStorageUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/util/CloudObjectStorageUtil.java @@ -77,6 +77,7 @@ public static String generateObjectName(String userId, @NonNull String taskId, @ SimpleDateFormat format = new SimpleDateFormat(DEFAULT_DATE_FORMAT); format.setTimeZone(TimeZone.getDefault()); builder.append(prefix).append(format.format(date)).append("/") + .append(StringUtils.isEmpty(userId) ? "" : userId + "/") .append(dateFormat.format(date)).append("/") .append(digest).append("/") .append(subPath).append("/") diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java index 0e76271252..914a5a7b81 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/OscService.java @@ -62,6 +62,7 @@ import com.oceanbase.odc.service.flow.task.model.OnlineSchemaChangeTaskResult; import com.oceanbase.odc.service.iam.HorizontalDataPermissionValidator; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.onlineschemachange.configuration.OnlineSchemaChangeProperties; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeScheduleTaskResult; import com.oceanbase.odc.service.onlineschemachange.model.OscLockDatabaseUserInfo; @@ -70,6 +71,7 @@ import com.oceanbase.odc.service.onlineschemachange.model.SwapTableType; import com.oceanbase.odc.service.onlineschemachange.model.UpdateRateLimiterConfigRequest; import com.oceanbase.odc.service.onlineschemachange.oscfms.ActionScheduler; +import com.oceanbase.odc.service.onlineschemachange.rename.LockTableSupportDecider; import com.oceanbase.odc.service.onlineschemachange.rename.OscDBUserUtil; import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.service.schedule.ScheduleTaskService; @@ -117,6 +119,8 @@ public class OscService { private TaskService taskService; @Autowired private ActionScheduler actionScheduler; + @Autowired + private OnlineSchemaChangeProperties onlineSchemaChangeProperties; @SkipAuthorize("internal authenticated") @@ -305,6 +309,7 @@ private void checkPermission(Long flowInstanceId) { private boolean getLockUserIsRequired(Long connectionId) { ConnectionConfig decryptedConnConfig = connectionService.getForConnectionSkipPermissionCheck(connectionId); + return OscDBUserUtil.isLockUserRequired(decryptedConnConfig.getDialectType(), () -> { ConnectionSessionFactory factory = new DefaultConnectSessionFactory(decryptedConnConfig); @@ -321,6 +326,7 @@ private boolean getLockUserIsRequired(Long connectionId) { } } return version; - }); + }, () -> LockTableSupportDecider.createWithJsonArrayWithDefaultValue( + onlineSchemaChangeProperties.getSupportLockTableObVersionJson())); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/configuration/OnlineSchemaChangeProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/configuration/OnlineSchemaChangeProperties.java index d727808ff8..01e9286667 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/configuration/OnlineSchemaChangeProperties.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/configuration/OnlineSchemaChangeProperties.java @@ -35,6 +35,8 @@ public class OnlineSchemaChangeProperties { @NestedConfigurationProperty private OmsProperties oms; + // in json array["obv1", "obv*",....] + private String supportLockTableObVersionJson; private boolean enableFullVerify; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java index 3c3f0944f0..d54882a702 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableAction.java @@ -40,6 +40,7 @@ import com.oceanbase.odc.service.onlineschemachange.oscfms.action.oms.ProjectStepResultChecker.ProjectStepResult; import com.oceanbase.odc.service.onlineschemachange.oscfms.state.OscStates; import com.oceanbase.odc.service.onlineschemachange.rename.DefaultRenameTableInvoker; +import com.oceanbase.odc.service.onlineschemachange.rename.LockTableSupportDecider; import com.oceanbase.odc.service.session.DBSessionManageFacade; import lombok.extern.slf4j.Slf4j; @@ -100,7 +101,7 @@ public OscActionResult execute(OscActionContext context) throws Exception { taskParameters.getUid(), taskParameters.getOmsProjectId(), taskParameters.getDatabaseName(), lastResult.getCheckFailedTime(), 25000); - }); + }, this::getLockTableSupportDecider); defaultRenameTableInvoker.invoke(taskParameters, parameters); // rename table success, jump to clean resource state return new OscActionResult(OscStates.SWAP_TABLE.getState(), null, OscStates.CLEAN_RESOURCE.getState()); @@ -111,6 +112,11 @@ public OscActionResult execute(OscActionContext context) throws Exception { } } + protected LockTableSupportDecider getLockTableSupportDecider() { + String lockTableMatchers = onlineSchemaChangeProperties.getSupportLockTableObVersionJson(); + return LockTableSupportDecider.createWithJsonArrayWithDefaultValue(lockTableMatchers); + } + protected boolean checkOMSProject(OscActionContext context) { OnlineSchemaChangeScheduleTaskParameters taskParameter = context.getTaskParameter(); // get result diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/DefaultRenameTableInvoker.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/DefaultRenameTableInvoker.java index 181dd6740d..54b03ee252 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/DefaultRenameTableInvoker.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/DefaultRenameTableInvoker.java @@ -53,13 +53,19 @@ public class DefaultRenameTableInvoker implements RenameTableInvoker { * supply if oms project has all data replicated */ private final Supplier dataReplicatedSupplier; + /** + * supply if lock table can be supported + */ + private final Supplier lockTableSupportDeciderSupplier; public DefaultRenameTableInvoker(ConnectionProvider connectionProvider, DBSessionManageFacade dbSessionManageFacade, - Supplier dataReplicatedSupplier) { + Supplier dataReplicatedSupplier, + Supplier lockTableSupportDeciderSupplier) { this.connectionProvider = connectionProvider; this.dbSessionManageFacade = dbSessionManageFacade; this.dataReplicatedSupplier = dataReplicatedSupplier; + this.lockTableSupportDeciderSupplier = lockTableSupportDeciderSupplier; } @Override @@ -103,7 +109,8 @@ private boolean tryRenameTable(ConnectionProvider connectionProvider, List interceptors = new LinkedList<>(); LockRenameTableFactory lockRenameTableFactory = new LockRenameTableFactory(); RenameTableInterceptor lockInterceptor = - lockRenameTableFactory.generate(connectionSession, dbSessionManageFacade); + lockRenameTableFactory.generate(connectionSession, dbSessionManageFacade, + lockTableSupportDeciderSupplier); interceptors.add(lockInterceptor); interceptors.add(new ForeignKeyInterceptor(connectionSession)); RenameTableHandler renameTableHandler = RenameTableHandlers.getForeignKeyHandler(connectionSession); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/LockRenameTableFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/LockRenameTableFactory.java index 0f5856016d..cabab19c0e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/LockRenameTableFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/LockRenameTableFactory.java @@ -15,6 +15,8 @@ */ package com.oceanbase.odc.service.onlineschemachange.rename; +import java.util.function.Supplier; + import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.core.shared.PreConditions; @@ -32,14 +34,15 @@ public class LockRenameTableFactory { public RenameTableInterceptor generate(ConnectionSession connectionSession, - DBSessionManageFacade dbSessionManageFacade) { + DBSessionManageFacade dbSessionManageFacade, + Supplier lockTableSupportDeciderSupplier) { PreConditions.notNull(connectionSession, "connectionSession"); DialectType dialectType = connectionSession.getDialectType(); PreConditions.notNull(dialectType, "dialectType"); String obVersion = ConnectionSessionUtil.getVersion(connectionSession); PreConditions.notNull(obVersion, "obVersion"); - return OscDBUserUtil.isLockUserRequired(dialectType, () -> obVersion) + return OscDBUserUtil.isLockUserRequired(dialectType, () -> obVersion, lockTableSupportDeciderSupplier) ? new LockUserInterceptor(connectionSession, dbSessionManageFacade) : new LockTableInterceptor(connectionSession); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/LockTableSupportDecider.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/LockTableSupportDecider.java new file mode 100644 index 0000000000..bc1931bace --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/LockTableSupportDecider.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.onlineschemachange.rename; + +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; + +import org.apache.commons.collections4.CollectionUtils; + +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.common.util.StringUtils; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author longpeng.zlp + * @date 2024/12/2 16:30 + */ +@Slf4j +public class LockTableSupportDecider { + public static final LockTableSupportDecider DEFAULT_LOCK_TABLE_DECIDER = + new LockTableSupportDecider(Arrays.asList("4\\.2\\.[5-9].*")); + private final List lockTableVersionMatchers; + + private LockTableSupportDecider(List lockTableVersionMatchers) { + this.lockTableVersionMatchers = lockTableVersionMatchers; + } + + public boolean supportLockTable(String dbVersion) { + log.info("decide if version = {} support lock table with candidates = {}", dbVersion, lockTableVersionMatchers); + if (CollectionUtils.isEmpty(lockTableVersionMatchers)) { + // empty not support + return false; + } + for (String candidate : lockTableVersionMatchers) { + Pattern pattern = Pattern.compile(candidate); + if (pattern.matcher(dbVersion).find()) { + return true; + } + } + return false; + } + + /** + * create with array in matchers + * + * @param versionMatchers + * @return + */ + public static LockTableSupportDecider createWithList(List versionMatchers) { + return new LockTableSupportDecider(versionMatchers); + } + + /** + * create with string in format json array + * + * @param versionMatchersInJsonArray string in json array eg "["v1", "v2", ...]" + * @return if versionMatchersInJsonArray is empty return DEFAULT_LOCK_TABLE_DECIDER otherwise build + * with input string + */ + public static LockTableSupportDecider createWithJsonArrayWithDefaultValue(String versionMatchersInJsonArray) { + if (StringUtils.isBlank(versionMatchersInJsonArray)) { + return DEFAULT_LOCK_TABLE_DECIDER; + } + List ret = JsonUtils.fromJsonList(versionMatchersInJsonArray, String.class); + return new LockTableSupportDecider(ret); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/OscDBUserUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/OscDBUserUtil.java index d823de91b7..1c5c495131 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/OscDBUserUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/rename/OscDBUserUtil.java @@ -32,11 +32,12 @@ */ public class OscDBUserUtil { - public static boolean isLockUserRequired(DialectType dialectType, Supplier obVersion) { + public static boolean isLockUserRequired(DialectType dialectType, Supplier obVersion, + Supplier lockTableSupportDeciderSupplier) { String version = obVersion.get(); if (dialectType.isOBMysql()) { // version is null, or version less than 4.2.5 - return !(version != null && VersionUtils.isGreaterThanOrEqualsTo(version, "4.2.5")); + return !(version != null && lockTableSupportDeciderSupplier.get().supportLockTable(version)); } else if (dialectType == DialectType.OB_ORACLE) { return version != null && !VersionUtils.isGreaterThanOrEqualsTo(version, "4.0.0"); } else { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java index e5e282d173..21696b8baf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java @@ -47,6 +47,7 @@ import com.oceanbase.odc.service.onlineschemachange.ddl.TableNameDescriptorFactory; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeSqlType; +import com.oceanbase.odc.service.onlineschemachange.rename.LockTableSupportDecider; import com.oceanbase.odc.service.onlineschemachange.rename.OscDBUserUtil; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; import com.oceanbase.tools.dbbrowser.model.DBConstraintType; @@ -249,7 +250,8 @@ private void validateType(String sql, OnlineSchemaChangeSqlType actualType, } private void validateLockUser(DialectType dialectType, String obVersion, List lockUsers) { - if (OscDBUserUtil.isLockUserRequired(dialectType, () -> obVersion) && CollectionUtils.isEmpty(lockUsers)) { + if (OscDBUserUtil.isLockUserRequired(dialectType, () -> obVersion, + () -> LockTableSupportDecider.DEFAULT_LOCK_TABLE_DECIDER) && CollectionUtils.isEmpty(lockUsers)) { throw new BadRequestException(ErrorCodes.OscLockUserRequired, new Object[] {lockUsers}, "Current db version should lock user required, but parameters do not contains user to lock."); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 2e48a6b17d..e999cf5d2f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -23,7 +23,6 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -74,19 +73,22 @@ import com.oceanbase.odc.metadb.schedule.LatestTaskMappingRepository; import com.oceanbase.odc.metadb.schedule.ScheduleEntity; import com.oceanbase.odc.metadb.schedule.ScheduleRepository; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; import com.oceanbase.odc.service.collaboration.project.ProjectService; import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.util.SpringContextUtil; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.connection.model.QueryConnectionParams; import com.oceanbase.odc.service.dlm.DlmLimiterService; import com.oceanbase.odc.service.dlm.model.DataArchiveParameters; import com.oceanbase.odc.service.dlm.model.DataDeleteParameters; import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; import com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq; import com.oceanbase.odc.service.flow.model.FlowInstanceDetailResp; -import com.oceanbase.odc.service.flow.util.DescriptionGenerator; import com.oceanbase.odc.service.iam.OrganizationService; import com.oceanbase.odc.service.iam.ProjectPermissionValidator; import com.oceanbase.odc.service.iam.UserService; @@ -128,6 +130,7 @@ import com.oceanbase.odc.service.schedule.model.TriggerStrategy; import com.oceanbase.odc.service.schedule.model.UpdateScheduleReq; import com.oceanbase.odc.service.schedule.processor.ScheduleChangePreprocessor; +import com.oceanbase.odc.service.schedule.util.ScheduleDescriptionGenerator; import com.oceanbase.odc.service.sqlplan.model.SqlPlanParameters; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.exception.JobException; @@ -149,6 +152,9 @@ public class ScheduleService { @Autowired private ScheduleRepository scheduleRepository; + + @Autowired + private ScheduleTaskRepository scheduleTaskRepository; @Autowired private AuthenticationFacade authenticationFacade; @Autowired @@ -212,6 +218,9 @@ public class ScheduleService { @Autowired private ApprovalFlowClient approvalFlowService; + @Autowired + private ScheduleDescriptionGenerator descriptionGenerator; + private final ScheduleMapper scheduleMapper = ScheduleMapper.INSTANCE; @Transactional(rollbackFor = Exception.class) @@ -269,7 +278,9 @@ public ChangeScheduleResp changeSchedule(ScheduleChangeParams req) { entity.setName(req.getCreateScheduleReq().getName()); entity.setProjectId(req.getProjectId()); - DescriptionGenerator.generateScheduleDescription(req); + if (StringUtils.isEmpty(req.getCreateScheduleReq().getDescription())) { + descriptionGenerator.generateScheduleDescription(req); + } entity.setDescription(req.getCreateScheduleReq().getDescription()); entity.setJobParametersJson(JsonUtils.toJson(req.getCreateScheduleReq().getParameters())); entity.setTriggerConfigJson(JsonUtils.toJson(req.getCreateScheduleReq().getTriggerConfig())); @@ -308,7 +319,8 @@ public ChangeScheduleResp changeSchedule(ScheduleChangeParams req) { validateTriggerConfig(req.getUpdateScheduleReq().getTriggerConfig()); } if (req.getOperationType() == OperationType.UPDATE - && (targetSchedule.getStatus() != ScheduleStatus.PAUSE || hasRunningTask(targetSchedule.getId()))) { + && (targetSchedule.getStatus() != ScheduleStatus.PAUSE + || hasExecutingTask(targetSchedule.getId()))) { log.warn("Update schedule is not allowed,status={}", targetSchedule.getStatus()); throw new IllegalStateException("Update schedule is not allowed."); } @@ -602,10 +614,6 @@ private boolean hasExecutingTask(Long scheduleId) { } private boolean isValidSchedule(Schedule schedule) { - - if (schedule.getStatus() != ScheduleStatus.ENABLED) { - return false; - } // check project if (schedule.getProjectId() != null) { try { @@ -814,6 +822,9 @@ public int getEnabledScheduleCountByProjectId(@NonNull Long projectId) { public Page listScheduleOverview(@NotNull Pageable pageable, @NotNull QueryScheduleParams params) { log.info("List schedule overview req:{}", params); + if (StringUtils.isNotEmpty(params.getId()) && !StringUtils.isNumeric(params.getId())) { + return Page.empty(); + } if (StringUtils.isNotBlank(params.getCreator())) { Set creatorIds = userService.getUsersByFuzzyNameWithoutPermissionCheck( params.getCreator()).stream().map(User::getId).collect(Collectors.toSet()); @@ -822,24 +833,21 @@ public Page listScheduleOverview(@NotNull Pageable pageable, } params.setCreatorIds(creatorIds); } - if (params.getDataSourceIds() == null) { - params.setDataSourceIds(new HashSet<>()); - } - if (StringUtils.isNotEmpty(params.getClusterId())) { - List datasourceIdsByCluster = connectionService.innerListIdByOrganizationIdAndClusterId( - authenticationFacade.currentOrganizationId(), params.getClusterId()); - if (datasourceIdsByCluster.isEmpty()) { - return Page.empty(); - } - params.getDataSourceIds().addAll(datasourceIdsByCluster); - } - if (StringUtils.isNotEmpty(params.getTenantId())) { - List datasourceIdsByTenantId = connectionService.innerListIdByOrganizationIdAndTenantId( - authenticationFacade.currentOrganizationId(), params.getTenantId()); - if (datasourceIdsByTenantId.isEmpty()) { + if (!CollectionUtils.isEmpty(params.getDataSourceIds()) || StringUtils.isNotEmpty(params.getClusterId()) + || StringUtils.isNotEmpty(params.getTenantId()) || StringUtils.isNotEmpty(params.getDataSourceName())) { + QueryConnectionParams datasourceParams = QueryConnectionParams.builder() + .ids(params.getDataSourceIds()) + .clusterNames(Collections.singletonList(params.getClusterId())) + .tenantNames(Collections.singletonList(params.getTenantId())) + .name(params.getDataSourceName()) + .build(); + Set datasourceIds = connectionService.listSkipPermissionCheck(datasourceParams).stream().map( + ConnectionConfig::getId).collect( + Collectors.toSet()); + if (datasourceIds.isEmpty()) { return Page.empty(); } - params.getDataSourceIds().addAll(datasourceIdsByTenantId); + params.setDataSourceIds(datasourceIds); } // load project by unique identifier if project id is null if (params.getProjectId() == null && StringUtils.isNotEmpty(params.getProjectUniqueIdentifier())) { @@ -863,13 +871,6 @@ public Page listScheduleOverview(@NotNull Pageable pageable, Page returnValue = scheduleRepository.find(pageable, params); List schedules = returnValue.getContent(); - if (params.getTriggerStrategy() != null) { - schedules = schedules.stream().filter(schedule -> { - TriggerConfig triggerConfig = JsonUtils.fromJson(schedule.getTriggerConfigJson(), TriggerConfig.class); - return triggerConfig.getTriggerStrategy().equals(params.getTriggerStrategy()); - }).collect(Collectors.toList()); - } - Map id2Overview = scheduleResponseMapperFactory.generateScheduleOverviewListMapper(schedules); @@ -887,29 +888,12 @@ public Page listScheduleTaskOverview(@NotNull Pageable pag return scheduleTaskService.getScheduleTaskListResp(pageable, scheduleId); } - public Page listScheduleTaskOverviewByScheduleType(@NotNull Pageable pageable, + public Page listScheduleTaskListOverview(@NotNull Pageable pageable, @NotNull QueryScheduleTaskParams params) { log.info("List schedule task overview req, params={}", params); - if (params.getDataSourceIds() == null) { - params.setDataSourceIds(new HashSet<>()); - } - if (StringUtils.isNotEmpty(params.getClusterId())) { - List datasourceIdsByCluster = connectionService.innerListIdByOrganizationIdAndClusterId( - authenticationFacade.currentOrganizationId(), params.getClusterId()); - if (datasourceIdsByCluster.isEmpty()) { - return Page.empty(); - } - params.getDataSourceIds().addAll(datasourceIdsByCluster); - } - if (StringUtils.isNotEmpty(params.getTenantId())) { - List datasourceIdsByTenantId = connectionService.innerListIdByOrganizationIdAndTenantId( - authenticationFacade.currentOrganizationId(), params.getTenantId()); - if (datasourceIdsByTenantId.isEmpty()) { - return Page.empty(); - } - params.getDataSourceIds().addAll(datasourceIdsByTenantId); + if (StringUtils.isNotEmpty(params.getId()) && !StringUtils.isNumeric(params.getId())) { + return Page.empty(); } - if (authenticationFacade.currentOrganization().getType() == OrganizationType.TEAM) { Set projectIds = params.getProjectId() == null ? projectService.getMemberProjectIds(authenticationFacade.currentUserId()) @@ -919,28 +903,26 @@ public Page listScheduleTaskOverviewByScheduleType(@No } params.setProjectIds(projectIds); } - params.setOrganizationId(authenticationFacade.currentOrganizationId()); - QueryScheduleParams scheduleParams = QueryScheduleParams.builder() .id(params.getScheduleId()) .name(params.getScheduleName()) .dataSourceIds(params.getDataSourceIds()) .databaseName(params.getDatabaseName()) .type(params.getScheduleType()) - .creator(params.getCreator()) - .projectId(params.getProjectId()) + .creatorIds(params.getCreatorIds()) + .projectIds(params.getProjectIds()) .organizationId(authenticationFacade.currentOrganizationId()) .build(); - - List scheduleList = scheduleRepository.find(scheduleParams).stream() - .map(scheduleMapper::entityToModel) - .collect(Collectors.toList()); - if (scheduleList.isEmpty()) { + Set scheduleIds = scheduleRepository.find(Pageable.unpaged(), scheduleParams).getContent() + .stream().map(ScheduleEntity::getId).collect(Collectors.toSet()); + if (scheduleIds.isEmpty()) { return Page.empty(); } - params.setSchedules(scheduleList); - - return scheduleTaskService.getConditionalScheduleTaskListResp(pageable, params); + params.setScheduleIds(scheduleIds); + Page returnValue = scheduleTaskRepository.find(pageable, params); + Map taskId2Overview = + scheduleResponseMapperFactory.generateScheduleTaskOverviewListMapper(returnValue.getContent()); + return returnValue.map(o -> taskId2Overview.get(o.getId())); } public List getAsyncDownloadUrl(Long id, List objectIds) { @@ -1057,10 +1039,6 @@ public Optional getLatestTask(Long id) { return Optional.of(res.get(0)); } - public boolean hasRunningTask(Long id) { - return false; - } - public void terminateByDatasourceIds(Set datasourceIds) { Set scheduleIds = scheduleRepository.getEnabledScheduleByConnectionIds(datasourceIds).stream() .map(ScheduleEntity::getId).collect( diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskConfiguration.java index 21028e8015..e2ed108cbd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskConfiguration.java @@ -23,6 +23,8 @@ import com.oceanbase.odc.service.schedule.alarm.ScheduleAlarmClient; import com.oceanbase.odc.service.schedule.flowtask.ApprovalFlowClient; import com.oceanbase.odc.service.schedule.flowtask.NoApprovalFlowClient; +import com.oceanbase.odc.service.schedule.util.DefaultScheduleDescriptionGenerator; +import com.oceanbase.odc.service.schedule.util.ScheduleDescriptionGenerator; /** * @Author：tinker @@ -45,4 +47,10 @@ public ApprovalFlowClient approvalFlowService() { public ScheduleAlarmClient scheduleAlarmClient() { return new DefaultScheduleAlarmClient(); } + + @Bean + @ConditionalOnMissingBean(ScheduleDescriptionGenerator.class) + public ScheduleDescriptionGenerator scheduleDescriptionGenerator() { + return new DefaultScheduleDescriptionGenerator(); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java index 53ba9f63a8..9435aaed0d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java @@ -15,13 +15,9 @@ */ package com.oceanbase.odc.service.schedule; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Function; import java.util.stream.Collectors; import org.quartz.JobKey; @@ -34,7 +30,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import com.alibaba.fastjson.JSON; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.shared.constant.ResourceType; @@ -43,15 +38,11 @@ import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.core.shared.exception.UnexpectedException; import com.oceanbase.odc.core.shared.exception.UnsupportedException; -import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.metadb.iam.UserRepository; import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; import com.oceanbase.odc.metadb.schedule.ScheduleTaskSpecs; -import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.metadb.task.JobRepository; -import com.oceanbase.odc.service.common.model.InnerUser; -import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.logicaldatabase.LogicalDatabaseChangeService; import com.oceanbase.odc.service.dispatch.DispatchResponse; import com.oceanbase.odc.service.dispatch.RequestDispatcher; @@ -66,20 +57,14 @@ import com.oceanbase.odc.service.schedule.model.DataArchiveRollbackParameters; import com.oceanbase.odc.service.schedule.model.QuartzKeyGenerator; import com.oceanbase.odc.service.schedule.model.QueryScheduleTaskParams; -import com.oceanbase.odc.service.schedule.model.Schedule; import com.oceanbase.odc.service.schedule.model.ScheduleTask; import com.oceanbase.odc.service.schedule.model.ScheduleTaskDetailResp; -import com.oceanbase.odc.service.schedule.model.ScheduleTaskListOverview; -import com.oceanbase.odc.service.schedule.model.ScheduleTaskListOverviewMapper; import com.oceanbase.odc.service.schedule.model.ScheduleTaskMapper; import com.oceanbase.odc.service.schedule.model.ScheduleTaskOverview; import com.oceanbase.odc.service.schedule.model.ScheduleTaskOverviewMapper; import com.oceanbase.odc.service.schedule.model.ScheduleTaskType; -import com.oceanbase.odc.service.schedule.model.ScheduleType; import com.oceanbase.odc.service.schedule.model.TriggerConfig; import com.oceanbase.odc.service.schedule.model.TriggerStrategy; -import com.oceanbase.odc.service.sqlplan.model.SqlPlanAttributes; -import com.oceanbase.odc.service.sqlplan.model.SqlPlanTaskResult; import com.oceanbase.odc.service.task.config.TaskFrameworkEnabledProperties; import com.oceanbase.odc.service.task.exception.JobException; import com.oceanbase.odc.service.task.model.ExecutorInfo; @@ -288,64 +273,9 @@ public Page getScheduleTaskListResp(Pageable pageable, Lon return list(pageable, scheduleId).map(ScheduleTaskOverviewMapper::map); } - public Page getConditionalScheduleTaskListResp(Pageable pageable, + public Page listEntity(Pageable pageable, QueryScheduleTaskParams params) { - Map scheduleMap = params.getSchedules().stream() - .collect(Collectors.toMap(schedule -> schedule.getId().toString(), Function.identity())); - - Specification specification = - Specification.where(ScheduleTaskSpecs.jobNameIn(scheduleMap.keySet())) - .and(ScheduleTaskSpecs.jobIdEquals(params.getScheduleId())) - .and(ScheduleTaskSpecs.statusIn(params.getStatuses())) - .and(ScheduleTaskSpecs.fireTimeLate(params.getStartTime())) - .and(ScheduleTaskSpecs.fireTimeBefore(params.getEndTime())); - - Page scheduleTaskPage = scheduleTaskRepository.findAll(specification, pageable).map( - scheduleTaskMapper::entityToModel); - - if (scheduleTaskPage.isEmpty()) { - return Page.empty(); - } - - // get creator info - Set creatorIds = params.getSchedules().stream().map(Schedule::getCreatorId).collect(Collectors.toSet()); - Map> users = userRepository.findByIdIn(creatorIds).stream().collect( - Collectors.groupingBy(UserEntity::getId)); - - // get database info - Set databaseIds = params.getSchedules().stream().map(Schedule::getDatabaseId).collect(Collectors.toSet()); - Map databaseMap = scheduleResponseMapperFactory.getDatabaseInfoByIds(databaseIds).stream() - .collect(Collectors.toMap(Database::getId, Function.identity())); - - // get job result json - List jobIds = scheduleTaskPage.getContent().stream().map(ScheduleTask::getJobId).filter(Objects::nonNull) - .collect(Collectors.toList()); - - Map resultMap = new HashMap<>(); - List jobEntities = jobRepository.findAllById(jobIds); - // only not null result is collected - for (JobEntity jobEntity : jobEntities) { - String resultJson = JobUtils.retrieveJobResultStr(jobEntity); - if (null != resultJson) { - resultMap.put(jobEntity.getId(), resultJson); - } - } - - return scheduleTaskPage.map(task -> { - Schedule schedule = scheduleMap.get(task.getJobName()); - ScheduleTaskListOverview overview = ScheduleTaskListOverviewMapper.map(task); - overview.setScheduleName(schedule.getName()); - overview.setCreator(new InnerUser(users.get(schedule.getCreatorId()).get(0), null)); - if (schedule.getType() == ScheduleType.SQL_PLAN) { - SqlPlanAttributes attribute = new SqlPlanAttributes(); - attribute.setDatabaseInfo(databaseMap.get(schedule.getDatabaseId())); - attribute.setTaskResult(JsonUtils.fromJson(resultMap.get(task.getJobId()), SqlPlanTaskResult.class)); - Map id2Attributes = new HashMap<>(); - id2Attributes.put(task.getId(), JsonUtils.toJson(attribute)); - overview.setAttributes(JSON.parseObject(id2Attributes.get(task.getId()))); - } - return overview; - }); + return scheduleTaskRepository.find(pageable, params); } public List listByJobNames(Set jobNames) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java index cb2c6e0331..06db052a84 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java @@ -24,6 +24,7 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; @@ -42,8 +43,11 @@ import com.oceanbase.odc.metadb.schedule.LatestTaskMappingEntity; import com.oceanbase.odc.metadb.schedule.LatestTaskMappingRepository; import com.oceanbase.odc.metadb.schedule.ScheduleEntity; +import com.oceanbase.odc.metadb.schedule.ScheduleRepository; import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; +import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.metadb.task.JobRepository; import com.oceanbase.odc.service.collaboration.project.ProjectService; import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.model.InnerUser; @@ -66,11 +70,14 @@ import com.oceanbase.odc.service.schedule.model.ScheduleOverview; import com.oceanbase.odc.service.schedule.model.ScheduleOverviewAttributes; import com.oceanbase.odc.service.schedule.model.ScheduleOverviewHist; +import com.oceanbase.odc.service.schedule.model.ScheduleTaskListOverview; import com.oceanbase.odc.service.schedule.model.ScheduleTaskParameters; +import com.oceanbase.odc.service.schedule.model.ScheduleTaskType; import com.oceanbase.odc.service.schedule.model.ScheduleType; import com.oceanbase.odc.service.schedule.model.TriggerConfig; import com.oceanbase.odc.service.sqlplan.model.SqlPlanAttributes; import com.oceanbase.odc.service.sqlplan.model.SqlPlanParameters; +import com.oceanbase.odc.service.sqlplan.model.SqlPlanTaskResult; import lombok.NonNull; @@ -105,6 +112,10 @@ public class ScheduleResponseMapperFactory { @Autowired private ApprovalPermissionService approvalPermissionService; @Autowired + private ScheduleRepository scheduleRepository; + @Autowired + private JobRepository jobRepository; + @Autowired private ProjectService projectService; @@ -184,6 +195,65 @@ public Map generateScheduleOverviewListMapper( }).collect(Collectors.toMap(ScheduleOverview::getScheduleId, o -> o)); } + public Map generateScheduleTaskOverviewListMapper( + Collection taskEntities) { + if (taskEntities.isEmpty()) { + return Collections.emptyMap(); + } + + Set scheduleIds = taskEntities.stream().map(o -> Long.parseLong(o.getJobName())).collect( + Collectors.toSet()); + + Map id2Schedule = scheduleRepository.findAllById(scheduleIds).stream().collect( + Collectors.toMap(ScheduleEntity::getId, o -> o)); + + // get creator info + Set creatorIds = + id2Schedule.values().stream().map(ScheduleEntity::getCreatorId).collect(Collectors.toSet()); + Map> users = userRepository.findByUserIds(creatorIds).stream().collect( + Collectors.groupingBy(UserEntity::getId)); + + // get database info + Set databaseIds = + id2Schedule.values().stream().map(ScheduleEntity::getDatabaseId).collect(Collectors.toSet()); + Map id2Database = getDatabaseByIds(databaseIds).stream() + .collect(Collectors.toMap(Database::getId, Function.identity())); + + // get job result json + List jobIds = taskEntities.stream().map(ScheduleTaskEntity::getJobId).filter(Objects::nonNull) + .collect(Collectors.toList()); + + Map resultMap = jobRepository.findAllById(jobIds).stream() + .filter(jobEntity -> jobEntity.getResultJson() != null) + .collect(Collectors.toMap(JobEntity::getId, JobEntity::getResultJson)); + + Map scheduleId2Attributes = generateAttributes( + id2Schedule.values()); + return taskEntities.stream().map(task -> { + ScheduleTaskListOverview t = new ScheduleTaskListOverview(); + ScheduleEntity schedule = id2Schedule.get(Long.parseLong(task.getJobName())); + t.setId(task.getId()); + t.setScheduleId(schedule.getId().toString()); + t.setStatus(task.getStatus()); + t.setType(ScheduleTaskType.valueOf(task.getJobGroup())); + t.setLastExecutionTime(task.getFireTime()); + t.setCreateTime(task.getCreateTime()); + t.setUpdateTime(task.getUpdateTime()); + t.setScheduleName(schedule.getName()); + t.setCreator(new InnerUser(users.get(schedule.getCreatorId()).get(0), null)); + String attributesJson; + if (t.getType() == ScheduleTaskType.SQL_PLAN) { + SqlPlanAttributes attributes = (SqlPlanAttributes) scheduleId2Attributes.get(schedule.getId()); + attributes.setTaskResult(JsonUtils.fromJson(resultMap.get(task.getJobId()), SqlPlanTaskResult.class)); + attributesJson = JsonUtils.toJson(attributes); + } else { + attributesJson = JsonUtils.toJson(scheduleId2Attributes.get(schedule.getId())); + } + t.setAttributes(JSON.parseObject(attributesJson)); + return t; + }).collect(Collectors.toMap(ScheduleTaskListOverview::getId, o -> o)); + } + @Deprecated public ScheduleDetailRespHist generateHistoryScheduleDetail(Schedule schedule) { ScheduleDetailRespHist resp = new ScheduleDetailRespHist(); @@ -297,9 +367,6 @@ public Map generateHistoryScheduleList(@NonNull Coll }).collect(Collectors.toMap(ScheduleOverviewHist::getId, o -> o)); } - public List getDatabaseInfoByIds(Set databaseIds) { - return getDatabaseByIds(databaseIds); - } private Map generateAttributes(Collection schedules) { Map> type2Entity = schedules.stream().collect( @@ -379,8 +446,9 @@ private List getDatabaseByIds(Set ids) { Set connectionIds = databases.stream().filter(e -> e.getDataSource() != null && e.getDataSource().getId() != null) .map(e -> e.getDataSource().getId()).collect(Collectors.toSet()); - Map id2Connection = dataSourceService.innerListByIds(connectionIds) - .stream().collect(Collectors.toMap(ConnectionConfig::getId, o -> o)); + Map id2Connection = + dataSourceService.internalListSkipUserCheck(connectionIds, false, false) + .stream().collect(Collectors.toMap(ConnectionConfig::getId, o -> o)); databases.forEach(database -> { if (id2Connection.containsKey(database.getDataSource().getId())) { database.setDataSource(id2Connection.get(database.getDataSource().getId())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java index b28bf8c51b..392abd3f72 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java @@ -71,6 +71,7 @@ private void executeInTaskFramework(JobExecutionContext context) { : dataDeleteParameters.getTargetDatabaseId())); parameters.getSourceDs().setQueryTimeout(dataDeleteParameters.getQueryTimeout()); parameters.getTargetDs().setQueryTimeout(dataDeleteParameters.getQueryTimeout()); + parameters.setShardingStrategy(dataDeleteParameters.getShardingStrategy()); Long jobId = publishJob(parameters, dataDeleteParameters.getTimeoutMillis(), dataDeleteParameters.getDatabaseId()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/QueryScheduleParams.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/QueryScheduleParams.java index fe2865c351..210637b6ee 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/QueryScheduleParams.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/QueryScheduleParams.java @@ -33,7 +33,8 @@ public class QueryScheduleParams { private Set dataSourceIds; - private Long id; + private String dataSourceName; + private String id; private String name; private List statuses; private ScheduleType type; @@ -48,6 +49,6 @@ public class QueryScheduleParams { private String databaseName; private String tenantId; private String clusterId; - private TriggerStrategy triggerStrategy; + private String triggerStrategy; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/QueryScheduleTaskParams.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/QueryScheduleTaskParams.java index 14ba60e5dc..f6355f56a0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/QueryScheduleTaskParams.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/QueryScheduleTaskParams.java @@ -27,9 +27,9 @@ @Data @Builder public class QueryScheduleTaskParams { - private Long id; - private Long scheduleId; - private List schedules; + private String id; + // it will be merged into scheduleIds when it is not null + private String scheduleId; private String scheduleName; private Set dataSourceIds; private List statuses; @@ -37,10 +37,14 @@ public class QueryScheduleTaskParams { private Date startTime; private Date endTime; private String creator; + private Set creatorIds; private Long projectId; private Set projectIds; private Long organizationId; private String databaseName; private String tenantId; private String clusterId; + + // inner use + private Set scheduleIds; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleTaskListOverview.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleTaskListOverview.java index be88d18fc1..95e2d77785 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleTaskListOverview.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleTaskListOverview.java @@ -18,15 +18,15 @@ import java.util.Date; import java.util.Map; -import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.common.model.InnerUser; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data -public class ScheduleTaskListOverview { +public class ScheduleTaskListOverview extends ScheduleTaskOverview { - private Long id; private String scheduleId; @@ -34,14 +34,6 @@ public class ScheduleTaskListOverview { private InnerUser creator; - private ScheduleTaskType type; - - private TaskStatus status; - - private Date createTime; - - private Date updateTime; - private Date lastExecutionTime; private Map attributes; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/util/DefaultScheduleDescriptionGenerator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/util/DefaultScheduleDescriptionGenerator.java new file mode 100644 index 0000000000..28df2c871a --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/util/DefaultScheduleDescriptionGenerator.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.schedule.util; + +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.service.schedule.model.ScheduleChangeParams; + +/** + * @Author：tinker + * @Date: 2024/11/11 11:35 + * @Descripition: + */ +public class DefaultScheduleDescriptionGenerator implements ScheduleDescriptionGenerator { + @Override + public void generateScheduleDescription(ScheduleChangeParams req) { + if (StringUtils.isEmpty(req.getCreateScheduleReq().getDescription())) { + String environmentName = req.getEnvironmentName(); + String connectionName = req.getConnectionName(); + String databaseName = req.getDatabaseName(); + String description = + StringUtils.isEmpty(connectionName) ? String.format("[%s]%s", environmentName, databaseName) + : String.format("[%s]%s.%s", environmentName, connectionName, databaseName); + req.getCreateScheduleReq().setDescription(description); + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/util/ScheduleDescriptionGenerator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/util/ScheduleDescriptionGenerator.java new file mode 100644 index 0000000000..b32e954795 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/util/ScheduleDescriptionGenerator.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.schedule.util; + +import com.oceanbase.odc.service.schedule.model.ScheduleChangeParams; + +/** + * @Author：tinker + * @Date: 2024/11/11 11:33 + * @Descripition: + */ +public interface ScheduleDescriptionGenerator { + + void generateScheduleDescription(ScheduleChangeParams req); + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index 9e8a1b0760..6c7d63ec1d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -113,9 +113,10 @@ public boolean start() throws Exception { } catch (Exception e) { log.warn("Failed to sync target table structure,table will be ignored,tableName={}", dlmTableUnit.getTableName(), e); - // jobStore.updateDlmTableUnitStatus(dlmTableUnit.getDlmTableUnitId(), TaskStatus.FAILED); - finishTableUnit(dlmTableUnitId, TaskStatus.FAILED); - continue; + if (!parameters.getSyncTableStructure().isEmpty()) { + finishTableUnit(dlmTableUnitId, TaskStatus.FAILED); + continue; + } } } try { diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableActionTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableActionTest.java index 668605ce9d..b7e446f6c9 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableActionTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/oscfms/action/oms/OmsSwapTableActionTest.java @@ -50,6 +50,7 @@ import com.oceanbase.odc.service.onlineschemachange.oscfms.action.oms.ProjectStepResultChecker.ProjectStepResult; import com.oceanbase.odc.service.onlineschemachange.oscfms.state.OscStates; import com.oceanbase.odc.service.onlineschemachange.rename.DefaultRenameTableInvoker; +import com.oceanbase.odc.service.onlineschemachange.rename.LockTableSupportDecider; import com.oceanbase.odc.service.onlineschemachange.rename.RenameTableHandler; import com.oceanbase.odc.service.onlineschemachange.rename.RenameTableHandlers; import com.oceanbase.odc.service.session.DBSessionManageFacade; @@ -120,7 +121,7 @@ public void testDefaultRenameTableNotInvoked() { private void doInvoke(DBSchemaAccessor dbSchemaAccessor, MockRenameTableHandler mockRenameTableHandler) { ConnectionSession connectionSession = Mockito.mock(ConnectionSession.class); Mockito.when(connectionSession.getDialectType()).thenReturn(DialectType.OB_MYSQL); - Mockito.when(connectionSession.getAttribute(ConnectionSessionConstants.OB_VERSION)).thenReturn("3.4.0"); + Mockito.when(connectionSession.getAttribute(ConnectionSessionConstants.OB_VERSION)).thenReturn("4.2.5.0"); Mockito.when(connectionSession.getSyncJdbcExecutor( ArgumentMatchers.anyString())).thenReturn(Mockito.mock(SyncJdbcExecutor.class)); ConnectionProvider connectionProvider = new ConnectionProvider() { @@ -137,7 +138,7 @@ public ConnectionSession createConnectionSession() { DefaultRenameTableInvoker renameTableInvoker = new DefaultRenameTableInvoker(connectionProvider, dbSessionManageFacade, - () -> true); + () -> true, () -> LockTableSupportDecider.DEFAULT_LOCK_TABLE_DECIDER); OnlineSchemaChangeParameters parameters = OscTestUtil.createOscParameters(); parameters.setSwapTableNameRetryTimes(1); try (MockedStatic mockedStatic = Mockito.mockStatic(DBSchemaAccessors.class); diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/rename/LockTableSupportDeciderTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/rename/LockTableSupportDeciderTest.java new file mode 100644 index 0000000000..b0b4b4d1bd --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/rename/LockTableSupportDeciderTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.onlineschemachange.rename; + +import java.util.Arrays; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +/** + * @author longpeng.zlp + * @date 2024/12/2 17:34 + */ +public class LockTableSupportDeciderTest { + @Test + public void testLockTableSupportDeciderNormal() { + List deciderList = Arrays.asList( + LockTableSupportDecider.createWithList(Arrays.asList("4\\.2\\.5.*", "5\\.2\\.5.*")), + LockTableSupportDecider + .createWithJsonArrayWithDefaultValue("[\"4\\\\.2\\\\.5.*\", \"5\\\\.2\\\\.5\"]")); + for (LockTableSupportDecider lockTableSupportDecider : deciderList) { + Assert.assertTrue(lockTableSupportDecider.supportLockTable("4.2.5")); + Assert.assertTrue(lockTableSupportDecider.supportLockTable("4.2.5.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.2.4.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.2.6.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.3.0.1")); + } + } + + @Test + public void testLockTableSupportDeciderWithDefault() { + List deciderList = Arrays.asList( + LockTableSupportDecider.createWithJsonArrayWithDefaultValue("")); + for (LockTableSupportDecider lockTableSupportDecider : deciderList) { + Assert.assertTrue(lockTableSupportDecider.supportLockTable("4.2.5")); + Assert.assertTrue(lockTableSupportDecider.supportLockTable("4.2.5.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.2.4.1")); + Assert.assertTrue(lockTableSupportDecider.supportLockTable("4.2.6.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.3.0.1")); + } + } + + @Test + public void testLockTableSupportDeciderRange() { + List deciderList = Arrays.asList( + LockTableSupportDecider.createWithList(Arrays.asList("4\\.2\\.[5-9].*", "5\\.2\\.[5-9].*")), + LockTableSupportDecider + .createWithJsonArrayWithDefaultValue("[\"4\\\\.2\\\\.[5-9].*\", \"5\\\\.2\\\\.[5-9].*\"]")); + for (LockTableSupportDecider lockTableSupportDecider : deciderList) { + Assert.assertTrue(lockTableSupportDecider.supportLockTable("4.2.5")); + Assert.assertTrue(lockTableSupportDecider.supportLockTable("4.2.5.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.2.4.1")); + Assert.assertTrue(lockTableSupportDecider.supportLockTable("4.2.6.1")); + Assert.assertTrue(lockTableSupportDecider.supportLockTable("4.2.7.1")); + Assert.assertTrue(lockTableSupportDecider.supportLockTable("4.2.8.1")); + Assert.assertTrue(lockTableSupportDecider.supportLockTable("4.2.9.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.2.10.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.3.0.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.3.1.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("3.3.1.1")); + } + } + + @Test + public void testLockTableSupportDeciderWithEmptyCollection() { + List deciderList = Arrays.asList( + LockTableSupportDecider.createWithList(Arrays.asList()), + LockTableSupportDecider.createWithJsonArrayWithDefaultValue("[]")); + for (LockTableSupportDecider lockTableSupportDecider : deciderList) { + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.2.5")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.2.5.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.2.4.1")); + Assert.assertFalse(lockTableSupportDecider.supportLockTable("4.2.6.1")); + } + } +} diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/rename/OscDBUserUtilTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/rename/OscDBUserUtilTest.java index 7ecd99d59e..a4663a90ff 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/rename/OscDBUserUtilTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/rename/OscDBUserUtilTest.java @@ -28,7 +28,9 @@ public class OscDBUserUtilTest { @Test public void testIsLockUserRequired() { - Assert.assertFalse(OscDBUserUtil.isLockUserRequired(DialectType.OB_MYSQL, () -> "4.2.5")); - Assert.assertTrue(OscDBUserUtil.isLockUserRequired(DialectType.OB_MYSQL, () -> "4.2.0")); + Assert.assertFalse(OscDBUserUtil.isLockUserRequired(DialectType.OB_MYSQL, () -> "4.2.5", + () -> LockTableSupportDecider.DEFAULT_LOCK_TABLE_DECIDER)); + Assert.assertTrue(OscDBUserUtil.isLockUserRequired(DialectType.OB_MYSQL, () -> "4.2.0", + () -> LockTableSupportDecider.DEFAULT_LOCK_TABLE_DECIDER)); } } From 826e4c2caf5771e3f5ca724aaf0336d6d2894218 Mon Sep 17 00:00:00 2001 From: guowl3 Date: Mon, 23 Dec 2024 14:05:01 +0800 Subject: [PATCH 062/118] fix(changelog): opt the changelog content (#4083) * fix change log * fix change log * code format --- .../schedule/ScheduleChangeLogService.java | 52 ++++++++++++++++++- .../odc/service/schedule/ScheduleService.java | 29 +++++++---- 2 files changed, 71 insertions(+), 10 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleChangeLogService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleChangeLogService.java index 29b8e9c61a..b147751cb5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleChangeLogService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleChangeLogService.java @@ -15,12 +15,17 @@ */ package com.oceanbase.odc.service.schedule; +import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.parser.Feature; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.NotFoundException; @@ -50,9 +55,20 @@ public ScheduleChangeLog createChangeLog(ScheduleChangeLog changeLog) { } public ScheduleChangeLog getByIdAndScheduleId(Long id, Long scheduleId) { - return scheduleChangeLogRepository.findByIdAndScheduleId(id, scheduleId).map(mapper::entityToModel) + ScheduleChangeLog changeLog = scheduleChangeLogRepository.findByIdAndScheduleId(id, scheduleId).map( + mapper::entityToModel) .orElseThrow(() -> new NotFoundException( ResourceType.ODC_SCHEDULE_CHANGELOG, "id", id)); + if (StringUtils.isNotEmpty(changeLog.getNewParameter()) + && StringUtils.isNotEmpty(changeLog.getPreviousParameters())) { + JSONObject pre = JSONObject.parseObject(changeLog.getPreviousParameters()); + JSONObject curr = JSONObject.parseObject(changeLog.getNewParameter()); + removeCommonKeys(pre, curr); + changeLog.setPreviousParameters(pre.toJSONString()); + changeLog.setNewParameter(curr.toJSONString()); + return changeLog; + } + return changeLog; } public ScheduleChangeLog getByFlowInstanceId(Long flowInstanceId) { @@ -78,4 +94,38 @@ public boolean hasApprovingChangeLog(Long scheduleId) { return listByScheduleId(scheduleId).stream().map(ScheduleChangeLog::getStatus) .anyMatch(ScheduleChangeStatus.APPROVING::equals); } + + + public static void removeCommonKeys(JSONObject json1, JSONObject json2) { + Iterator keys = json1.keySet().iterator(); + while (keys.hasNext()) { + String key = keys.next(); + + if (json2.containsKey(key)) { + Object value1 = json1.get(key); + Object value2 = json2.get(key); + + if (isJsonString(value1.toString()) && isJsonString(value2.toString())) { + JSONObject nestedJson1 = JSON.parseObject(value1.toString()); + JSONObject nestedJson2 = JSON.parseObject(value2.toString()); + removeCommonKeys(nestedJson1, nestedJson2); + + json1.put(key, nestedJson1); + json2.put(key, nestedJson2); + } else if (value1.equals(value2)) { + keys.remove(); + json2.remove(key); + } + } + } + } + + public static boolean isJsonString(String value) { + try { + JSON.parseObject(value, Feature.IgnoreNotMatch); + return true; + } catch (Exception e) { + return false; + } + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index e999cf5d2f..6e8baa762f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -52,6 +52,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; +import com.alibaba.fastjson.JSONObject; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.alarm.AlarmUtils; @@ -366,16 +367,26 @@ private ScheduleChangeLog createScheduleChangelog(ScheduleChangeParams req, Sche "Concurrent change schedule request is not allowed"); } + String pre = null; + String curr = null; + if (req.getOperationType() == OperationType.UPDATE) { + JSONObject preJsonObject = new JSONObject(); + preJsonObject.put("triggerConfig", targetSchedule.getTriggerConfig()); + preJsonObject.put("parameters", targetSchedule.getParameters()); + pre = preJsonObject.toJSONString(); + JSONObject currJsonOBject = new JSONObject(); + currJsonOBject.put("triggerConfig", req.getUpdateScheduleReq().getTriggerConfig()); + currJsonOBject.put("parameters", req.getUpdateScheduleReq().getParameters()); + curr = currJsonOBject.toJSONString(); + } else if (req.getOperationType() == OperationType.CREATE) { + JSONObject currJsonOBject = new JSONObject(); + currJsonOBject.put("triggerConfig", req.getCreateScheduleReq().getTriggerConfig()); + currJsonOBject.put("parameters", req.getCreateScheduleReq().getParameters()); + curr = currJsonOBject.toJSONString(); + } + ScheduleChangeLog changeLog = scheduleChangeLogService.createChangeLog( - ScheduleChangeLog.build(targetSchedule.getId(), req.getOperationType(), - req.getOperationType() == OperationType.UPDATE - ? JsonUtils.toJson(targetSchedule.getParameters()) - : null, - req.getOperationType() == OperationType.UPDATE - ? JsonUtils.toJson(req.getUpdateScheduleReq().getParameters()) - : req.getOperationType() == OperationType.CREATE - ? JsonUtils.toJson(req.getCreateScheduleReq().getParameters()) - : null, + ScheduleChangeLog.build(targetSchedule.getId(), req.getOperationType(), pre, curr, ScheduleChangeStatus.APPROVING)); log.info("Create change log success,changLog={}", changeLog); req.setScheduleChangeLogId(changeLog.getId()); From ea5fa0436ca9b99a926c2c8d7dddf5cabcdb7b02 Mon Sep 17 00:00:00 2001 From: guowl3 Date: Mon, 23 Dec 2024 16:00:40 +0800 Subject: [PATCH 063/118] opt error message (#4086) --- .../odc/core/shared/PreConditions.java | 10 ---------- .../odc/core/shared/constant/ErrorCodes.java | 3 +++ .../resources/i18n/ErrorMessages.properties | 5 ++++- .../i18n/ErrorMessages_zh_CN.properties | 6 +++++- .../i18n/ErrorMessages_zh_TW.properties | 7 ++++++- .../odc/service/schedule/ScheduleService.java | 20 +++++++++---------- 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/PreConditions.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/PreConditions.java index 30282f7a3b..6a88790ecc 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/PreConditions.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/PreConditions.java @@ -269,16 +269,6 @@ public static boolean validFileSuffix(String fileName, List safeSuffixLi "file suffix is illegal"); } - public static void validNoDuplicatedAlterSchedule( - String parameterName, Object parameterValue, BooleanSupplier duplicatedChecker) { - if (duplicatedChecker.getAsBoolean()) { - throw new BadRequestException(ErrorCodes.AlterScheduleExists, - new Object[] {parameterName, parameterValue}, - String.format("alter task already exists by %s=%s", parameterName, - parameterValue.toString())); - } - } - public static void validSingleton(Collection collection, String parameterName) { notEmpty(collection, parameterName); if (collection.size() != 1) { diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java index 8fa9eefb6c..ecd738b5b5 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java @@ -170,6 +170,9 @@ public enum ErrorCodes implements ErrorCode { // Schedule AlterScheduleExists, InvalidCronExpression, + UpdateNotAllowed, + PauseNotAllowed, + DeleteNotAllowed, // Partition plan PartitionPlanNoDropPreviewSqlGenerated, diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties index 85f4974381..b2479eebe8 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties @@ -200,4 +200,7 @@ com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch=The time type precisi com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=Database access is denied because the current user does not have {0} permissions for the database. com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=Query profile is only available for OceanBase Database with versions equal to or higher than {0}. com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=Someone has just modified this file. Please refresh the page to get the latest version and continue editing. -com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=The database user type used in the workspace must be a super account. \ No newline at end of file +com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=The database user type used in the workspace must be a super account. +com.oceanbase.odc.ErrorCodes.UpdateNotAllowed=Editing is not allowed in the current state. Please try again after disabling. +com.oceanbase.odc.ErrorCodes.PauseNotAllowed=Disabling is not allowed in the current state, please check if there are any records in execution. +com.oceanbase.odc.ErrorCodes.DeleteNotAllowed=Deletion is not allowed in the current state. diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties index c4ea9bdadd..e910200748 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties @@ -199,4 +199,8 @@ com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch=时间类型精度不 com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=数据库访问被拒绝，因为当前用户没有数据库的{0}权限 com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=执行详情仅在高于或等于 {0} 的 OceanBase 版本可用 com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=有人刚刚修改了这个工作簿，请刷新页面以获取最新版本并重新编辑 -com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空间中使用的数据库用户类型必须为超级账号 \ No newline at end of file +com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空间中使用的数据库用户类型必须为超级账号 +com.oceanbase.odc.ErrorCodes.UpdateNotAllowed=当前状态下不允许编辑，请禁用后重试 +com.oceanbase.odc.ErrorCodes.PauseNotAllowed=当前状态下不允许禁用，请检查是否存在执行中的记录 +com.oceanbase.odc.ErrorCodes.DeleteNotAllowed=当前状态下不允许删除 + diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties index 5e8ed7b042..6b05e76c53 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties @@ -199,4 +199,9 @@ com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch=時間類型精度不 com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=資料庫存取被拒絕，因為目前使用者沒有資料庫的{0}權限 com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=執行詳情僅在高於或等於 {0} 的 OceanBase 版本可用 com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=有人剛剛修改了這個工作簿，請重新整理頁面以取得最新版本並繼續編輯 -com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空間中使用的資料庫用戶類型必須為超級帳號 \ No newline at end of file +com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空間中使用的資料庫用戶類型必須為超級帳號 +com.oceanbase.odc.ErrorCodes.UpdateNotAllowed=當前狀態下不允許編輯，請在禁用後重試 +com.oceanbase.odc.ErrorCodes.PauseNotAllowed=當前狀態下不允許禁用，請檢查是否存在執行中的記錄 +com.oceanbase.odc.ErrorCodes.DeleteNotAllowed=在目前狀態下不允許删除 + + diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 6e8baa762f..12748c93a6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -318,19 +318,17 @@ public ChangeScheduleResp changeSchedule(ScheduleChangeParams req) { targetSchedule = nullSafeGetByIdWithCheckPermission(req.getScheduleId(), true); if (req.getOperationType() == OperationType.UPDATE) { validateTriggerConfig(req.getUpdateScheduleReq().getTriggerConfig()); + PreConditions.validRequestState(targetSchedule.getStatus() == ScheduleStatus.PAUSE, + ErrorCodes.UpdateNotAllowed, null, "Update schedule is not allowed."); } - if (req.getOperationType() == OperationType.UPDATE - && (targetSchedule.getStatus() != ScheduleStatus.PAUSE - || hasExecutingTask(targetSchedule.getId()))) { - log.warn("Update schedule is not allowed,status={}", targetSchedule.getStatus()); - throw new IllegalStateException("Update schedule is not allowed."); + if (req.getOperationType() == OperationType.PAUSE) { + PreConditions.validRequestState(!hasExecutingTask(targetSchedule.getId()), ErrorCodes.PauseNotAllowed, + null, "Pause schedule is not allowed."); } - if (req.getOperationType() == OperationType.DELETE - && targetSchedule.getStatus() != ScheduleStatus.TERMINATED - && targetSchedule.getStatus() != ScheduleStatus.COMPLETED) { - log.warn("Delete schedule is not allowed,status={}", targetSchedule.getStatus()); - throw new IllegalStateException( - "Delete schedule is not allowed, only can delete terminated schedule or finished schedule."); + if (req.getOperationType() == OperationType.DELETE) { + PreConditions.validRequestState(targetSchedule.getStatus() == ScheduleStatus.TERMINATED + || targetSchedule.getStatus() == ScheduleStatus.COMPLETED, ErrorCodes.DeleteNotAllowed, null, + "Delete schedule is not allowed."); } } From cc2a2e18b13d5c9cb22182aeab4bd0ba225b7199 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Tue, 24 Dec 2024 10:20:41 +0800 Subject: [PATCH 064/118] fix(flow):view export function can be initiated without permission #4090 --- .../com/oceanbase/odc/service/flow/FlowInstanceService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java index 4e1985d504..9328db7877 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java @@ -769,7 +769,7 @@ private void checkCreateFlowInstancePermission(CreateFlowInstanceReq req) { } else if (CollectionUtils.isNotEmpty(parameters.getExportDbObjects())) { ConnectionConfig config = connectionService.getBasicWithoutPermissionCheck(req.getConnectionId()); parameters.getExportDbObjects().forEach(item -> { - if (item.getDbObjectType() == ObjectType.TABLE) { + if (item.getDbObjectType() == ObjectType.TABLE || item.getDbObjectType() == ObjectType.VIEW) { resource2Types.put( DBResource.from(config, req.getDatabaseName(), item.getObjectName(), ResourceType.ODC_TABLE), From 4fafc7a994c5d88c94c4651dccbcf54d87d0d207 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Tue, 24 Dec 2024 10:27:59 +0800 Subject: [PATCH 065/118] fix(project): archiving projects will fail because of wrong check of tickets references #4089 --- .../collaboration/ProjectServiceTest.java | 8 +++++ .../collaboration/project/ProjectService.java | 36 +++++++++++++++---- .../odc/service/schedule/ScheduleService.java | 2 +- .../schedule/model/ScheduleStatus.java | 2 +- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java index f40330ffd1..75319561a1 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java @@ -63,6 +63,7 @@ import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.iam.model.User; import com.oceanbase.odc.service.iam.model.UserResourceRole; +import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.test.tool.TestRandom; /** @@ -100,6 +101,9 @@ public class ProjectServiceTest extends ServiceTestEnv { @MockBean private ProjectPermissionValidator projectPermissionValidator; + @MockBean + private ScheduleService scheduleService; + @Before public void setUp() { Mockito.when(userService.nullSafeGet(Mockito.anyLong())).thenReturn(getUserEntity()); @@ -181,6 +185,8 @@ public void testArchiveProject_Archived() throws InterruptedException { doNothing().when(projectPermissionValidator).checkProjectRole(anyCollection(), anyList()); SetArchivedReq req = new SetArchivedReq(); req.setArchived(true); + Mockito.when(scheduleService.listUnfinishedSchedulesByProjectId(Pageable.unpaged(), saved.getId())) + .thenReturn(Page.empty()); Project archived = projectService.setArchived(saved.getId(), req); Assert.assertTrue(archived.getArchived()); } @@ -201,6 +207,8 @@ public void testDeleteProject_ArchivedProject_Success() throws InterruptedExcept doNothing().when(projectPermissionValidator).checkProjectRole(anyCollection(), anyList()); SetArchivedReq req = new SetArchivedReq(); req.setArchived(true); + Mockito.when(scheduleService.listUnfinishedSchedulesByProjectId(Pageable.unpaged(), saved.getId())) + .thenReturn(Page.empty()); projectService.setArchived(saved.getId(), req); Assert.assertTrue(projectService.batchDelete(new HashSet<>(Arrays.asList(saved.getId())))); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java index 3b8deaf288..bc27f02819 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java @@ -31,6 +31,7 @@ import javax.validation.constraints.NotNull; import org.apache.commons.collections4.CollectionUtils; +import org.quartz.SchedulerException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; @@ -88,6 +89,8 @@ import com.oceanbase.odc.service.iam.model.User; import com.oceanbase.odc.service.iam.model.UserResourceRole; import com.oceanbase.odc.service.schedule.ScheduleService; +import com.oceanbase.odc.service.schedule.model.ScheduleOverviewHist; +import com.oceanbase.odc.service.schedule.model.ScheduleType; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -428,12 +431,16 @@ public Boolean batchDelete(Set projectIds) { @Transactional(rollbackFor = Exception.class) @SkipAuthorize("Internal usage") - public TicketReference getProjectTicketReference(Long projectId) { + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) + public TicketReference getProjectTicketReference(@NonNull Long projectId) { TicketReference reference = new TicketReference(); reference.setUnfinishedFlowInstances( flowInstanceService.listUnfinishedFlowInstances(Pageable.unpaged(), projectId).getContent()); reference.setUnfinishedSchedules( - scheduleService.listUnfinishedSchedulesByProjectId(Pageable.unpaged(), projectId).getContent()); + scheduleService.listUnfinishedSchedulesByProjectId(Pageable.unpaged(), projectId).getContent().stream() + .filter(schedule -> schedule.getType() != ScheduleType.PARTITION_PLAN).collect( + Collectors.toList())); return reference; } @@ -556,14 +563,31 @@ public void checkCurrentUserProjectRolePermissions(@NotNull Project project, } public void checkUnfinishedTickets(@NonNull Long projectId) { - if (scheduleService.getEnabledScheduleCountByProjectId(projectId) > 0) { - throw new BadRequestException( - "There exists unfinished schedule tasks in the project, please disable them before archiving the project."); - } if (flowInstanceService.listUnfinishedFlowInstances(Pageable.unpaged(), projectId).hasContent()) { throw new BadRequestException( "There exists unfinished tickets in the project, please stop them before archiving the project."); } + List schedules = + scheduleService.listUnfinishedSchedulesByProjectId(Pageable.unpaged(), projectId).getContent(); + // no unfinished schedules + if (CollectionUtils.isEmpty(schedules)) { + return; + } + // There exists unfinished schedules(except for partition plans) in the project + if (schedules.stream().anyMatch(schedule -> schedule.getType() != ScheduleType.PARTITION_PLAN)) { + throw new BadRequestException( + "There exists unfinished schedule tasks in the project, please stop them before archiving the project."); + } + // Terminate all partition plans if exists + schedules.stream().filter(schedule -> schedule.getType() == ScheduleType.PARTITION_PLAN) + .forEach(schedule -> { + try { + scheduleService.innerTerminate(schedule.getId()); + } catch (SchedulerException e) { + log.warn("Failed to terminate partition plan schedule, scheduleId={}", schedule.getId()); + throw new RuntimeException(e); + } + }); } private Project entityToModel(ProjectEntity entity, List userResourceRoles) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 12748c93a6..9ee901eca3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -820,7 +820,7 @@ public Page list(@NotNull Pageable pageable, @NotNull Quer public Page listUnfinishedSchedulesByProjectId(@NonNull Pageable pageable, @NonNull Long projectId) { - return list(pageable, QueryScheduleParams.builder().projectId(projectId) + return list(pageable, QueryScheduleParams.builder().projectIds(Collections.singleton(projectId)) .statuses(ScheduleStatus.listUnfinishedStatus()).build()); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java index ddad70d69b..4f27d246f8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java @@ -43,6 +43,6 @@ public enum ScheduleStatus { DELETED; public static List listUnfinishedStatus() { - return Collections.unmodifiableList(Arrays.asList(CREATING, APPROVING, PAUSE, ENABLED)); + return Collections.unmodifiableList(Arrays.asList(CREATING, APPROVING, ENABLED)); } } From 0ff95aebcd4619704ceeba852fadfa09e5137b41 Mon Sep 17 00:00:00 2001 From: guowl3 Date: Tue, 24 Dec 2024 11:05:21 +0800 Subject: [PATCH 066/118] fix(approval): approval is not needed in individual organizations #4067 --- .../oceanbase/odc/service/schedule/ScheduleService.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 9ee901eca3..e3a8fb33b5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -388,7 +388,13 @@ private ScheduleChangeLog createScheduleChangelog(ScheduleChangeParams req, Sche ScheduleChangeStatus.APPROVING)); log.info("Create change log success,changLog={}", changeLog); req.setScheduleChangeLogId(changeLog.getId()); - Long approvalFlowInstanceId = approvalFlowService.create(req); + Long approvalFlowInstanceId; + if (organizationService.get(targetSchedule.getId()).isPresent() + && organizationService.get(targetSchedule.getId()).get().getType() == OrganizationType.INDIVIDUAL) { + approvalFlowInstanceId = null; + } else { + approvalFlowInstanceId = approvalFlowService.create(req); + } if (approvalFlowInstanceId != null) { changeLog.setFlowInstanceId(approvalFlowInstanceId); scheduleChangeLogService.updateFlowInstanceIdById(changeLog.getId(), approvalFlowInstanceId); From 60bf81ad896e6c1ac7b9096e80be02b7e0e41698 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Tue, 24 Dec 2024 17:33:43 +0800 Subject: [PATCH 067/118] fix(saml): saml may blocked tomcat thread --- .../saml/SamlRegistrationConfigHelper.java | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlRegistrationConfigHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlRegistrationConfigHelper.java index bc11acfcfe..2bba0244d6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlRegistrationConfigHelper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlRegistrationConfigHelper.java @@ -21,7 +21,10 @@ import static com.oceanbase.odc.service.integration.util.EncryptionUtil.PRIVATE_KEY_SUFFIX; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; import java.security.KeyFactory; import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; @@ -33,6 +36,11 @@ import java.util.function.Supplier; import org.springframework.boot.context.properties.PropertyMapper; +import org.springframework.core.io.DefaultResourceLoader; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; +import org.springframework.core.io.UrlResource; +import org.springframework.security.saml2.Saml2Exception; import org.springframework.security.saml2.core.Saml2X509Credential; import org.springframework.security.saml2.core.Saml2X509Credential.Saml2X509CredentialType; import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistration; @@ -47,14 +55,18 @@ import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; import com.oceanbase.odc.service.integration.saml.SamlParameter.Signing; +import lombok.Setter; + public final class SamlRegistrationConfigHelper { + private static final ResourceLoader resourceLoader = new DefaultResourceLoader(); + public static RelyingPartyRegistration asRegistration(SSOIntegrationConfig ssoIntegrationConfig) { Verify.verify("SAML".equals(ssoIntegrationConfig.getType()), "Invalid type=" + ssoIntegrationConfig.getType()); SamlParameter parameter = (SamlParameter) ssoIntegrationConfig.getSsoParameter(); boolean usingMetadata = StringUtils.hasText(parameter.getMetadataUri()); Builder builder = (usingMetadata) - ? RelyingPartyRegistrations.fromMetadataLocation(parameter.getMetadataUri()) + ? fromMetadataLocation(parameter.getMetadataUri()) .registrationId(parameter.getRegistrationId()) : RelyingPartyRegistration.withRegistrationId(parameter.getRegistrationId()); builder.assertionConsumerServiceLocation(parameter.getAcsLocation()); @@ -74,6 +86,22 @@ public static RelyingPartyRegistration asRegistration(SSOIntegrationConfig ssoIn return registration; } + public static RelyingPartyRegistration.Builder fromMetadataLocation(String metadataLocation) { + Resource resource = resourceLoader.getResource(metadataLocation); + if (resource instanceof UrlResource) { + UrlResource urlResource = (UrlResource) resource; + resource = new TimeoutUrlResourceAdaptor(urlResource.getURL()); + } + try (InputStream source = resource.getInputStream()) { + return RelyingPartyRegistrations.fromMetadata(source); + } catch (IOException ex) { + if (ex.getCause() instanceof Saml2Exception) { + throw (Saml2Exception) ex.getCause(); + } + throw new Saml2Exception(ex); + } + } + private static void addCredentialIfNotNull(Collection credentials, Supplier supplier) { Saml2X509Credential saml2X509Credential = supplier.get(); @@ -184,4 +212,21 @@ private static void validateSigningCredentials(SamlParameter parameter, boolean "Signing credentials must not be empty when authentication requests require signing."); } } + + @Setter + static class TimeoutUrlResourceAdaptor extends UrlResource { + + private int connectTimeout = 2000; + private int readTimeout = 2000; + + public TimeoutUrlResourceAdaptor(URL url) { + super(url); + } + + @Override + protected void customizeConnection(HttpURLConnection con) throws IOException { + con.setConnectTimeout(connectTimeout); + con.setReadTimeout(readTimeout); + } + } } From e48fe7b8727e06e70023457047bf3bde0560d1ce Mon Sep 17 00:00:00 2001 From: LioRoger Date: Tue, 24 Dec 2024 17:34:22 +0800 Subject: [PATCH 068/118] fix(task): correct region key name for resourceID --- .../oceanbase/odc/service/task/caller/K8sJobCaller.java | 4 ++-- .../odc/service/task/caller/ResourceIDUtil.java | 8 ++++---- .../odc/service/task/dummy/LocalMockK8sJobClient.java | 4 ++-- .../odc/service/task/util/JobPropertiesUtils.java | 9 +++++---- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java index d59345e510..973d14b5db 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java @@ -75,9 +75,9 @@ protected K8sResourceContext buildK8sResourceContext(JobContext context, Resourc protected ResourceLocation buildResourceLocation(JobContext context) { // TODO(tianke): confirm is this correct? String region = ResourceIDUtil.checkAndGetJobProperties(context.getJobProperties(), - ResourceIDUtil.DEFAULT_REGION_PROP_NAME, ResourceIDUtil.DEFAULT_PROP_VALUE); + ResourceIDUtil.REGION_PROP_NAME, ResourceIDUtil.DEFAULT_PROP_VALUE); String group = ResourceIDUtil.checkAndGetJobProperties(context.getJobProperties(), - ResourceIDUtil.DEFAULT_GROUP_PROP_NAME, ResourceIDUtil.DEFAULT_PROP_VALUE); + ResourceIDUtil.GROUP_PROP_NAME, ResourceIDUtil.DEFAULT_PROP_VALUE); return new ResourceLocation(region, group); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ResourceIDUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ResourceIDUtil.java index 80db91871e..c22d3d7617 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ResourceIDUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ResourceIDUtil.java @@ -32,8 +32,8 @@ */ @Slf4j public class ResourceIDUtil { - public static final String DEFAULT_REGION_PROP_NAME = "region"; - public static final String DEFAULT_GROUP_PROP_NAME = "cloudProvider"; + public static final String REGION_PROP_NAME = "regionName"; + public static final String GROUP_PROP_NAME = "cloudProvider"; public static final String DEFAULT_PROP_VALUE = "local"; /** @@ -70,8 +70,8 @@ public static String checkAndGetJobProperties(Map jobParameters, */ public static ResourceID getResourceID(ExecutorIdentifier executorIdentifier, Map jobProperties) { - String region = checkAndGetJobProperties(jobProperties, DEFAULT_REGION_PROP_NAME, DEFAULT_PROP_VALUE); - String group = checkAndGetJobProperties(jobProperties, DEFAULT_GROUP_PROP_NAME, DEFAULT_PROP_VALUE); + String region = checkAndGetJobProperties(jobProperties, REGION_PROP_NAME, DEFAULT_PROP_VALUE); + String group = checkAndGetJobProperties(jobProperties, GROUP_PROP_NAME, DEFAULT_PROP_VALUE); return new ResourceID(new ResourceLocation(region, group), DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE, executorIdentifier.getNamespace(), executorIdentifier.getExecutorName()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClient.java index 3fd2abd342..921ba8ab3b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClient.java @@ -76,8 +76,8 @@ private JobContext getJobContext(Object extraData) { @Override public Optional get(String namespace, String arn) throws JobException { - K8sPodResource ret = new K8sPodResource(ResourceIDUtil.DEFAULT_REGION_PROP_NAME, - ResourceIDUtil.DEFAULT_GROUP_PROP_NAME, DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE, + K8sPodResource ret = new K8sPodResource(ResourceIDUtil.REGION_PROP_NAME, + ResourceIDUtil.GROUP_PROP_NAME, DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE, namespace, arn, ResourceState.AVAILABLE, "127.0.0.1", new Date(System.currentTimeMillis())); return Optional.of(ret); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java index a0418ea523..124f323649 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java @@ -21,6 +21,7 @@ import com.oceanbase.odc.common.util.MapUtils; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.service.cloud.model.CloudProvider; +import com.oceanbase.odc.service.task.caller.ResourceIDUtil; import com.oceanbase.odc.service.task.enums.TaskMonitorMode; import lombok.NonNull; @@ -40,20 +41,20 @@ public static Map getLabels(@NonNull Map jobProp public static void setCloudProvider(@NonNull Map jobProperties, @NonNull CloudProvider cloudProvider) { - jobProperties.put("cloudProvider", cloudProvider.toString()); + jobProperties.put(ResourceIDUtil.GROUP_PROP_NAME, cloudProvider.toString()); } public static CloudProvider getCloudProvider(@NonNull Map jobProperties) { - String cloudProvider = jobProperties.get("cloudProvider"); + String cloudProvider = jobProperties.get(ResourceIDUtil.GROUP_PROP_NAME); return StringUtils.isBlank(cloudProvider) ? CloudProvider.NONE : CloudProvider.valueOf(cloudProvider); } public static void setRegionName(@NonNull Map jobProperties, @NonNull String regionName) { - jobProperties.put("regionName", regionName); + jobProperties.put(ResourceIDUtil.REGION_PROP_NAME, regionName); } public static String getRegionName(@NonNull Map jobProperties) { - return jobProperties.get("regionName"); + return jobProperties.get(ResourceIDUtil.REGION_PROP_NAME); } public static void setMonitorMode(@NonNull Map jobProperties, From 36fe6cc5501409e86f9b197881c38e79a5767631 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Wed, 25 Dec 2024 14:14:43 +0800 Subject: [PATCH 069/118] fix(db): cannot replace original pl name where editing pl (#4088) * Enhance regex escaping in DBPLModifyHelper * Enhance regex escaping in DBPLModifyHelper * Refactor test method name for clarity in StringUtilsTest --- .../odc/common/util/StringUtils.java | 27 +++++++++++++++++ .../odc/common/util/StringUtilsTest.java | 29 +++++++++++++++++++ .../odc/service/db/DBPLModifyHelper.java | 5 +++- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java index fe4acd2cfa..8c3eb20a96 100644 --- a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java @@ -21,6 +21,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -58,6 +59,7 @@ public abstract class StringUtils extends org.apache.commons.lang3.StringUtils { private static final Pattern PORT_PATTERN = Pattern.compile("^(([1-9]\\d{0,3}|[1-5]\\d{4}|6[0-4]\\d{3}|65[0-4]\\d{2}|655[0-2]\\d|6553" + "[0-5]))$"); + private static final char[] REGULAR_CHARS = {'.', '*', '+', '?', '^', '$', '[', ']', '(', ')', '{', '}', '|', '\\'}; private static final String ORACLE_ESCAPE_KEYWORD = "ESCAPE '\\'"; private static final String DEFAULT_VARIABLE_PREFIX = "${"; private static final String DEFAULT_VARIABLE_SUFFIX = "}"; @@ -66,6 +68,22 @@ public abstract class StringUtils extends org.apache.commons.lang3.StringUtils { private StringUtils() {} + public static Optional escapeRegex(String str) { + return Optional.ofNullable(str).map(s -> { + if (s.isEmpty()) { + return ""; + } + StringBuilder escapedString = new StringBuilder(); + for (char c : s.toCharArray()) { + if (isRegularChar(c)) { + escapedString.append('\\'); + } + escapedString.append(c); + } + return escapedString.toString(); + }); + } + public static Boolean checkMysqlIdentifierQuoted(final String str) { if (StringUtils.isBlank(str)) { return false; @@ -471,4 +489,13 @@ public static String getTranslatableKey(@NonNull String str) { } return str.substring(DEFAULT_VARIABLE_PREFIX.length(), str.length() - DEFAULT_VARIABLE_SUFFIX.length()); } + + private static boolean isRegularChar(char c) { + for (char special : REGULAR_CHARS) { + if (c == special) { + return true; + } + } + return false; + } } diff --git a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/StringUtilsTest.java b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/StringUtilsTest.java index 6501c5857d..2b6c1f4bfb 100644 --- a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/StringUtilsTest.java +++ b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/StringUtilsTest.java @@ -15,6 +15,8 @@ */ package com.oceanbase.odc.common.util; +import java.util.Optional; + import org.junit.Assert; import org.junit.Test; @@ -398,4 +400,31 @@ public void startWithIgnoreSpaceAndNewLines_targetsNonStartingWithPrefixWithMaxM Assert.assertTrue(StringUtils.startsWithIgnoreSpaceAndNewLines(target, prefix, true, 2)); } + @Test + public void escapeRegex_null_isNotPresent() { + Assert.assertFalse(StringUtils.escapeRegex(null).isPresent()); + } + + @Test + public void escapeRegex_stringContainsRegularChar_escapeSuccess() { + String regex = ".*+?^$[](){}|\\"; + String expect = "\\.\\*\\+\\?\\^\\$\\[\\]\\(\\)\\{\\}\\|\\\\"; + Optional escapedRegex = StringUtils.escapeRegex(regex); + Assert.assertTrue(escapedRegex.isPresent()); + Assert.assertEquals(expect, escapedRegex.get()); + } + + @Test + public void replaceFirst_replaceOriginalNameContainingRegularCharsInPlEdit_success() { + String pl = + "create function `FuN_test@111--#%&*()_+` (`p1` int(12)) returns int(11) begin declare v1 int; set v1 = p1 + 1; return v1; end;"; + String originalName = "FuN_test@111--#%&*()_+"; + String replaceName = "replace_name"; + String expect = + "create function `replace_name` (`p1` int(12)) returns int(11) begin declare v1 int; set v1 = p1 + 1; return v1; end;"; + Optional escapedRegex = StringUtils.escapeRegex(originalName); + String actual = pl.replaceFirst(escapedRegex.get(), replaceName); + Assert.assertEquals(expect, actual); + + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java index c08a07de84..24b1325e39 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java @@ -28,6 +28,7 @@ import org.springframework.stereotype.Component; import org.springframework.validation.annotation.Validated; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.VersionUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; @@ -99,7 +100,9 @@ private EditPLResp executeWrappedEditPL(String sessionId, EditPLReq editPLReq, DBObjectType plType, String tempPlName) throws Exception { String plName = editPLReq.getObjectName(); String editPLSql = editPLReq.getSql(); - String tempPLSql = editPLSql.replaceFirst(plName, tempPlName); + String escapeRegexPlName = StringUtils.escapeRegex(plName) + .orElseThrow(() -> new IllegalStateException(String.format("%s name cannot be null", plType))); + String tempPLSql = editPLSql.replaceFirst(escapeRegexPlName, tempPlName); StringBuilder wrappedSqlBuilder = new StringBuilder(); ConnectionSession connectionSession = sessionService.nullSafeGet(sessionId, true); SqlCommentProcessor processor = ConnectionSessionUtil.getSqlCommentProcessor(connectionSession); From 127a8df7a9801bb7e7581cc096a762de6fd7c45a Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Wed, 25 Dec 2024 14:21:45 +0800 Subject: [PATCH 070/118] fix(dbbrowser): cant get correct ddl of procedure (#4084) * Add show create procedure for MySQL schema accessor * Add test for MySQL procedure with DEFINER clause * Refactor MySQL procedure parsing and add tests * Remove redundant MySQL procedure parameter query --- .../MySQLNoLessThan5700SchemaAccessor.java | 29 ++- .../tools/dbbrowser/parser/PLParserTest.java | 166 ++++++++++++++++++ 2 files changed, 177 insertions(+), 18 deletions(-) diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java index bdeb69f07b..5fc03fca41 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java @@ -1333,31 +1333,24 @@ public DBProcedure getProcedure(String schemaName, String procedureName) { .value(schemaName) .append(" and ROUTINE_TYPE = 'PROCEDURE' and ROUTINE_NAME=") .value(procedureName); - - MySQLSqlBuilder queryForParameters = new MySQLSqlBuilder(); - queryForParameters.append( - "select PARAMETER_MODE, PARAMETER_NAME, DTD_IDENTIFIER from `information_schema`.`parameters` where SPECIFIC_SCHEMA=") - .value(schemaName) - .append(" and SPECIFIC_NAME=") - .value(procedureName) - .append(" and ROUTINE_TYPE='PROCEDURE'"); DBProcedure procedure = new DBProcedure(); procedure.setProName(procedureName); - MySQLSqlBuilder parameters = new MySQLSqlBuilder(); - - jdbcOperations.query(queryForParameters.toString(), (rs) -> { - parameters.append(rs.getString("PARAMETER_MODE")).space() - .identifier(rs.getString("PARAMETER_NAME")).space() - .append(rs.getString("DTD_IDENTIFIER")).append(","); + MySQLSqlBuilder getDDL = new MySQLSqlBuilder(); + getDDL.append("show create procedure "); + if (schemaName == null) { + getDDL.identifier(procedureName); + } else { + getDDL.identifier(schemaName); + getDDL.append("."); + getDDL.identifier(procedureName); + } + jdbcOperations.query(getDDL.toString(), (rs) -> { + procedure.setDdl(rs.getString("Create Procedure")); }); jdbcOperations.query(sql1.toString(), (rs) -> { procedure.setDefiner(rs.getString("DEFINER")); procedure.setCreateTime(Timestamp.valueOf(rs.getString("CREATED"))); procedure.setModifyTime(Timestamp.valueOf(rs.getString("LAST_ALTERED"))); - procedure.setDdl(String.format("create procedure %s (%s) %s;", - StringUtils.quoteMysqlIdentifier(procedure.getProName()), - StringUtils.substring(parameters.toString(), 0, parameters.length() - 1), - rs.getString("ROUTINE_DEFINITION"))); }); return parseProcedureDDL(procedure); } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java index 1e2114b9c9..63390a1a23 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java @@ -57,6 +57,172 @@ public void testParseMysqlProcedure() { Assert.assertEquals(1, result.getParamList().size()); } + @Test + public void testParseMysqlProcedureContainsDefiner() { + String pl = "create DEFINER = `root`@`%` PROCEDURE testProduce (out p1 int) \n" + "BEGIN \n" + + "DECLARE Eno INT DEFAULT 10000;\n" + + "DECLARE En VARCHAR(20);\n" + "DECLARE J VARCHAR(20);\n" + "DECLARE M INT DEFAULT 80000;\n" + + "DECLARE H YEAR;\n" + "DECLARE Dno INT;\n" + "DECLARE i INT DEFAULT 1; \n" + "RETURN;\n" + "END;"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(7, result.getVaribaleList().size()); + Assert.assertEquals("testProduce", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(1, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerWithoutParams() { + String pl = "CREATE DEFINER = `root`@`%` PROCEDURE simple_procedure()\n" + + "BEGIN\n" + + " SELECT 'Hello, World!';\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(0, result.getVaribaleList().size()); + Assert.assertEquals("simple_procedure", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(0, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerWithInput() { + String pl = "CREATE DEFINER = `root`@`%` PROCEDURE greet_user(IN user_name VARCHAR(50))\n" + + "BEGIN\n" + + " SELECT CONCAT('Hello, ', user_name, '!');\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(0, result.getVaribaleList().size()); + Assert.assertEquals("greet_user", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(1, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerWithInputAndOutput() { + String pl = "CREATE DEFINER = `root`@`%` PROCEDURE get_user_name(IN user_id INT, OUT user_name VARCHAR(50))\n" + + "BEGIN\n" + + " SELECT name INTO user_name FROM users WHERE id = user_id;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(0, result.getVaribaleList().size()); + Assert.assertEquals("get_user_name", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(2, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerAndCondition() { + String pl = + "CREATE DEFINER = `root`@`%` PROCEDURE check_user_status(IN user_id INT, OUT user_status VARCHAR(20))\n" + + + "BEGIN\n" + + " DECLARE user_count INT;\n" + + "\n" + + " SELECT COUNT(*) INTO user_count FROM users WHERE id = user_id;\n" + + "\n" + + " IF user_count > 0 THEN\n" + + " SELECT status INTO user_status FROM users WHERE id = user_id;\n" + + " ELSE\n" + + " SET user_status = 'User not found';\n" + + " END IF;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(1, result.getVaribaleList().size()); + Assert.assertEquals("check_user_status", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(2, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerAndFunction() { + String pl = "CREATE DEFINER = `root`@`%` PROCEDURE calculate_user_average_age(OUT average_age FLOAT)\n" + + "BEGIN\n" + + " SELECT AVG(age) INTO average_age FROM users;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(0, result.getVaribaleList().size()); + Assert.assertEquals("calculate_user_average_age", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(1, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerAndCursor() { + String pl = "CREATE DEFINER = `root`@`%` PROCEDURE list_all_users()\n" + + "BEGIN\n" + + " DECLARE done INT DEFAULT 0;\n" + + " DECLARE user_id INT;\n" + + " DECLARE user_name VARCHAR(50);\n" + + "\n" + + " DECLARE user_cursor CURSOR FOR SELECT id, name FROM users;\n" + + " DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1;\n" + + "\n" + + " OPEN user_cursor;\n" + + "\n" + + " read_loop: LOOP\n" + + " FETCH user_cursor INTO user_id, user_name;\n" + + " IF done THEN\n" + + " LEAVE read_loop;\n" + + " END IF;\n" + + " SELECT CONCAT('User ID: ', user_id, ', User Name: ', user_name);\n" + + " END LOOP;\n" + + "\n" + + " CLOSE user_cursor;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(5, result.getVaribaleList().size()); + Assert.assertEquals("list_all_users", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(0, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerAndException() { + String pl = + "CREATE DEFINER = `root`@`%` PROCEDURE safe_update_user_email(IN user_id INT, IN new_email VARCHAR(100), OUT result_message VARCHAR(100))\n" + + + "BEGIN\n" + + " DECLARE CONTINUE HANDLER FOR SQLEXCEPTION\n" + + " BEGIN\n" + + " SET result_message = 'An error occurred while updating the email.';\n" + + " END;\n" + + "\n" + + " UPDATE users SET email = new_email WHERE id = user_id;\n" + + " IF ROW_COUNT() = 0 THEN\n" + + " SET result_message = 'No user found with the provided ID.';\n" + + " ELSE\n" + + " SET result_message = 'Email updated successfully.';\n" + + " END IF;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(1, result.getVaribaleList().size()); + Assert.assertEquals("safe_update_user_email", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(3, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerAndMultipleTables() { + String pl = + "CREATE DEFINER = `root`@`%` PROCEDURE activate_user_and_log(IN user_id INT, OUT result_message VARCHAR(100))\n" + + + "BEGIN\n" + + " UPDATE users SET status = 'active' WHERE id = user_id;\n" + + " \n" + + " IF ROW_COUNT() > 0 THEN\n" + + " INSERT INTO logs (user_id, action, created_at) VALUES (user_id, 'Activated user', NOW());\n" + + + " SET result_message = 'User activated and logged.';\n" + + " ELSE\n" + + " SET result_message = 'User not found for activation.';\n" + + " END IF;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(0, result.getVaribaleList().size()); + Assert.assertEquals("activate_user_and_log", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(2, result.getParamList().size()); + } + @Test public void testParseOracleProcedure() { String pl = "create procedure pl_test(p1 in int default 1, p2 in varchar2) \n" + "is \n" + "v1 number;\n" From cad3676c412eddd91ca8b978de0d9785ec03e682 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Wed, 25 Dec 2024 14:22:21 +0800 Subject: [PATCH 071/118] fix(flow): add currentUserResourceRole in flow APIs #4096 --- .../odc/service/collaboration/project/ProjectService.java | 4 +++- .../odc/service/flow/model/FlowInstanceDetailResp.java | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java index bc27f02819..cff5ab0a6c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java @@ -338,7 +338,7 @@ public Page list(@Valid QueryProjectParams params, @NotNull Pageable pa @SkipAuthorize("odc internal usage") public List listByIds(@NotEmpty Set ids) { - return repository.findAllById(ids).stream().map(projectMapper::entityToModel).collect(Collectors.toList()); + return repository.findAllById(ids).stream().map(this::entityToModel).collect(Collectors.toList()); } private Page innerList(@Valid QueryProjectParams params, @NotNull Pageable pageable, @@ -610,6 +610,8 @@ private Project entityToModel(ProjectEntity entity) { Project project = projectMapper.entityToModel(entity); project.setCreator(currentInnerUser()); project.setLastModifier(currentInnerUser()); + project.setCurrentUserResourceRoles( + getProjectId2ResourceRoleNames().getOrDefault(project.getId(), Collections.EMPTY_SET)); return project; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java index 8b194eee11..e9f18dd262 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java @@ -176,6 +176,7 @@ public FlowInstanceDetailResp map(@NonNull FlowInstance flowInstance, @NonNull F resp.setStatus(flowInstance.getStatus()); resp.setDescription(flowInstance.getDescription()); resp.setProjectId(flowInstance.getProjectId()); + resp.setProject(getProjectById.apply(flowInstance.getProjectId())); List instances = flowInstance.filterInstanceNode(instance -> FlowNodeType.SERVICE_TASK == instance.getNodeType() From 1f184553f18a257e98737ab0a3d6ef29bf1b83ef Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Wed, 25 Dec 2024 14:28:41 +0800 Subject: [PATCH 072/118] fix(notification): support send account name in message (#4085) * do not validate webhook * support sending approver account name * support sending creator account name --- .../notification/constant/EventLabelKeys.java | 2 + .../helper/ChannelConfigValidator.java | 74 ++++++++----------- .../notification/helper/EventBuilder.java | 10 +++ 3 files changed, 41 insertions(+), 45 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java index 2db4ac8d8c..632dbafd12 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java @@ -50,8 +50,10 @@ public class EventLabelKeys { public static final String PROJECT_NAME = "projectName"; public static final String CREATOR_ID = "creatorId"; public static final String CREATOR_NAME = "creatorName"; + public static final String CREATOR_ACCOUNT = "creatorAccountName"; public static final String APPROVER_ID = "approverId"; public static final String APPROVER_NAME = "approverName"; + public static final String APPROVER_ACCOUNT = "approverAccountName"; /** * global info diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/ChannelConfigValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/ChannelConfigValidator.java index 7d2a05b67c..6026757b6a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/ChannelConfigValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/ChannelConfigValidator.java @@ -46,10 +46,7 @@ */ @Component public class ChannelConfigValidator { - private static final Pattern ILLEGAL_CHARACTERS_PATTERN = Pattern.compile("[@#;$,\\[\\]{}\\-\\\\^\"<>]"); - private static final String DINGTALK_WEBHOOK_PREFIX = "https://oapi.dingtalk.com/robot"; - private static final String FEISHU_WEBHOOK_PREFIX = "https://open.feishu.cn/open-apis/bot"; - private static final String WECOM_WEBHOOK_PREFIX = "https://qyapi.weixin.qq.com/cgi-bin/webhook"; + private static final Pattern ILLEGAL_CHARACTERS_PATTERN = Pattern.compile("[@#;$,\\[\\]{}\\\\^\"<>]"); @Autowired private IntegrationConfigProperties integrationConfigProperties; @@ -59,13 +56,13 @@ public class ChannelConfigValidator { public void validate(@NonNull ChannelType type, BaseChannelConfig channelConfig) { switch (type) { case DingTalk: - validateDingTalkChannelConfig((DingTalkChannelConfig) channelConfig); + validateWebhook(((DingTalkChannelConfig) channelConfig).getWebhook()); return; case WeCom: - validateWeComChannelConfig((WeComChannelConfig) channelConfig); + validateWebhook(((WeComChannelConfig) channelConfig).getWebhook()); return; case Feishu: - validateFeishuChannelConfig((WebhookChannelConfig) channelConfig); + validateWebhook(((WebhookChannelConfig) channelConfig).getWebhook()); return; case Webhook: validateWebhookChannelConfig((WebhookChannelConfig) channelConfig); @@ -75,45 +72,9 @@ public void validate(@NonNull ChannelType type, BaseChannelConfig channelConfig) } } - private void validateDingTalkChannelConfig(DingTalkChannelConfig channelConfig) { - Verify.notEmpty(channelConfig.getWebhook(), "webhook"); - Verify.verify(channelConfig.getWebhook().startsWith(DINGTALK_WEBHOOK_PREFIX), - "please input an valid Dingtalk webhook"); - } - - private void validateFeishuChannelConfig(WebhookChannelConfig channelConfig) { - Verify.notEmpty(channelConfig.getWebhook(), "webhook"); - Verify.verify(channelConfig.getWebhook().startsWith(FEISHU_WEBHOOK_PREFIX), - "please input an valid Feishu webhook"); - } - - private void validateWeComChannelConfig(WeComChannelConfig channelConfig) { - Verify.notEmpty(channelConfig.getWebhook(), "webhook"); - Verify.verify(channelConfig.getWebhook().startsWith(WECOM_WEBHOOK_PREFIX), - "please input an valid WeCom webhook"); - } - private void validateWebhookChannelConfig(WebhookChannelConfig channelConfig) { - Verify.notEmpty(channelConfig.getWebhook(), "webhook"); - Verify.verify( - channelConfig.getWebhook().startsWith("http://") || channelConfig.getWebhook().startsWith("https://"), - "Webhook should start with 'http://' or 'https://'"); - if (ILLEGAL_CHARACTERS_PATTERN.matcher(channelConfig.getWebhook()).find()) { - throw new IllegalArgumentException("Webhook contains illegal characters"); - } - try { - if (CollectionUtils.isNotEmpty(integrationConfigProperties.getUrlWhiteList())) { - Verify.verify(SSRFChecker.checkUrlInWhiteList(channelConfig.getWebhook(), - integrationConfigProperties.getUrlWhiteList()), - "The webhook is forbidden due to SSRF protection"); - } else { - Verify.verify(SSRFChecker.checkHostNotInBlackList(new URL(channelConfig.getWebhook()).getHost(), - notificationProperties.getHostBlackList()), - "The webhook is forbidden due to SSRF protection"); - } - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } + validateWebhook(channelConfig.getWebhook()); + String httpProxy = channelConfig.getHttpProxy(); Verify.verify(StringUtils.isEmpty(httpProxy) || httpProxy.split(":").length == 3, "Illegal http proxy, it should be like 'http(s)://host:port'"); @@ -135,4 +96,27 @@ private void validateWebhookChannelConfig(WebhookChannelConfig channelConfig) { } } + + private void validateWebhook(String webhook) { + Verify.notEmpty(webhook, "webhook"); + Verify.verify( + webhook.startsWith("http://") || webhook.startsWith("https://"), + "Webhook should start with 'http://' or 'https://'"); + if (ILLEGAL_CHARACTERS_PATTERN.matcher(webhook).find()) { + throw new IllegalArgumentException("Webhook contains illegal characters"); + } + try { + if (CollectionUtils.isNotEmpty(integrationConfigProperties.getUrlWhiteList())) { + Verify.verify(SSRFChecker.checkUrlInWhiteList(webhook, + integrationConfigProperties.getUrlWhiteList()), + "The webhook is forbidden due to SSRF protection"); + } else { + Verify.verify(SSRFChecker.checkHostNotInBlackList(new URL(webhook).getHost(), + notificationProperties.getHostBlackList()), + "The webhook is forbidden due to SSRF protection"); + } + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java index da26e3ae18..082d1a31e1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java @@ -15,10 +15,12 @@ */ package com.oceanbase.odc.service.notification.helper; +import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.APPROVER_ACCOUNT; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.APPROVER_ID; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.APPROVER_NAME; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.CLUSTER_NAME; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.CONNECTION_ID; +import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.CREATOR_ACCOUNT; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.CREATOR_ID; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.CREATOR_NAME; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.DATABASE_ID; @@ -155,6 +157,7 @@ public Event ofApprovedTask(TaskEntity task, Long approver) { Event event = ofTask(task, TaskEvent.APPROVED); if (approver == null) { event.getLabels().putIfNonNull(APPROVER_NAME, AUTO_APPROVAL_KEY); + event.getLabels().putIfNonNull(APPROVER_ACCOUNT, AUTO_APPROVAL_KEY); } else { event.getLabels().put(APPROVER_ID, approver + ""); } @@ -166,6 +169,7 @@ public Event ofRejectedTask(TaskEntity task, Long approver) { Event event = ofTask(task, TaskEvent.APPROVAL_REJECTION); if (approver == null) { event.getLabels().putIfNonNull(APPROVER_NAME, AUTO_APPROVAL_KEY); + event.getLabels().putIfNonNull(APPROVER_ACCOUNT, AUTO_APPROVAL_KEY); } else { event.getLabels().put(APPROVER_ID, approver + ""); } @@ -318,6 +322,7 @@ private void resolveLabels(EventLabels labels, T task) { try { UserEntity user = userService.nullSafeGet(labels.getLongFromString(CREATOR_ID)); labels.putIfNonNull(CREATOR_NAME, user.getName()); + labels.putIfNonNull(CREATOR_ACCOUNT, user.getAccountName()); } catch (Exception e) { log.warn("failed to query creator info.", e); } @@ -339,14 +344,19 @@ private void resolveLabels(EventLabels labels, T task) { try { if ("null".equals(labels.get(APPROVER_ID))) { labels.putIfNonNull(APPROVER_NAME, AUTO_APPROVAL_KEY); + labels.putIfNonNull(APPROVER_ACCOUNT, AUTO_APPROVAL_KEY); } else if (labels.get(APPROVER_ID).startsWith("[")) { List approverIds = JsonUtils.fromJsonList(labels.get(APPROVER_ID), Long.class); List approvers = userService.batchNullSafeGet(approverIds); labels.putIfNonNull(APPROVER_NAME, String.join(" | ", approvers.stream().map(User::getName).collect(Collectors.toSet()))); + labels.putIfNonNull(APPROVER_ACCOUNT, + String.join(" | ", + approvers.stream().map(User::getAccountName).collect(Collectors.toSet()))); } else { UserEntity user = userService.nullSafeGet(labels.getLongFromString(APPROVER_ID)); labels.putIfNonNull(APPROVER_NAME, user.getName()); + labels.putIfNonNull(APPROVER_ACCOUNT, user.getAccountName()); } } catch (Exception e) { log.warn("failed to query approver.", e); From 1668fe1cc08d2b76c06cb3396d9db5b92291e991 Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Wed, 25 Dec 2024 14:29:13 +0800 Subject: [PATCH 073/118] fix(execution-plan): avoid invalid number #4087 Open --- .../plugin/connect/obmysql/diagnose/PlanGraphBuilder.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphBuilder.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphBuilder.java index a05aa030d1..6dae6fb688 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphBuilder.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphBuilder.java @@ -26,6 +26,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.lang3.math.NumberUtils; + import com.oceanbase.odc.common.graph.Graph; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.shared.exception.UnexpectedException; @@ -83,7 +85,8 @@ private static void parsePlanByJsonMap(Map jsonMap, Graph graph, graph.insertVertex(operator); id2Operator.put(operator.getGraphId(), operator); if (!"-1".equals(parentId)) { - graph.insertEdge(id2Operator.get(parentId), operator, (int) jsonMap.get("EST.ROWS")); + int rows = NumberUtils.isDigits(jsonMap.get("EST.ROWS").toString()) ? (int) jsonMap.get("EST.ROWS") : 0; + graph.insertEdge(id2Operator.get(parentId), operator, rows); } operator.setStatus(QueryStatus.PREPARING); String name = (String) jsonMap.get("NAME"); From 60e13495cad7c9dbeff301adc31a5fdd3482ec04 Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Wed, 25 Dec 2024 15:23:31 +0800 Subject: [PATCH 074/118] fix(notification): fail to enqueue schedule event (#4091) * fix failed to enqueue event * fix incorrect task type * avoid NPE --- .../notification/helper/EventBuilder.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java index 082d1a31e1..11bfe37caa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java @@ -52,7 +52,9 @@ import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.constant.TaskType; +import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.core.shared.exception.UnexpectedException; import com.oceanbase.odc.metadb.flow.FlowInstanceEntity; import com.oceanbase.odc.metadb.flow.FlowInstanceRepository; @@ -86,6 +88,7 @@ import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter; import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter.ApplyTable; import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleParameters; +import com.oceanbase.odc.service.schedule.model.ScheduleChangeParams; import com.oceanbase.odc.service.schedule.model.ScheduleTask; import lombok.extern.slf4j.Slf4j; @@ -246,6 +249,19 @@ private Event ofTask(TaskEntity task, TaskEvent status) { : database.getEnvironment().getName(), database.getName())) .collect(Collectors.joining(","))); labels.putIfNonNull(PROJECT_ID, projectId); + } else if (task.getTaskType() == TaskType.ALTER_SCHEDULE) { + AlterScheduleParameters parameter = JsonUtils.fromJson(task.getParametersJson(), + AlterScheduleParameters.class); + ScheduleChangeParams scheduleChangeParams = parameter.getScheduleChangeParams(); + Verify.notNull(scheduleChangeParams, "scheduleChangeParams"); + ScheduleEntity schedule = scheduleRepository.findById(scheduleChangeParams.getScheduleId()) + .orElseThrow(() -> new NotFoundException(ResourceType.ODC_SCHEDULE, "id", + scheduleChangeParams.getScheduleId())); + projectId = schedule.getProjectId(); + labels.putIfNonNull(PROJECT_ID, projectId); + labels.putIfNonNull(DATABASE_ID, schedule.getDatabaseId()); + labels.putIfNonNull(DATABASE_NAME, schedule.getDatabaseName()); + labels.putIfNonNull(TASK_TYPE, schedule.getType().name()); } else { throw new UnexpectedException("task.databaseId should not be null"); } From 8b8edc5ea5a9b19ca026a73362b62bb1bebedc62 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Thu, 26 Dec 2024 17:59:25 +0800 Subject: [PATCH 075/118] fix NPE (#4106) fix(integration): external approval integration will cause NPE if hyperlinkExpression is null #4106 --- .../odc/service/flow/factory/FlowResponseMapperFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java index 18cbee0a9a..2166f8f0b9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java @@ -266,7 +266,8 @@ private FlowNodeInstanceMapper generateNodeMapper(@NonNull Collection flow IntegrationConfig config = integrationService.detailWithoutPermissionCheck(externalApproval.getApprovalId()); ApprovalProperties properties = ApprovalProperties.from(config); - if (StringUtils.isEmpty(properties.getAdvanced().getHyperlinkExpression())) { + if (Objects.isNull(properties.getAdvanced()) + || StringUtils.isEmpty(properties.getAdvanced().getHyperlinkExpression())) { return null; } TemplateVariables variables = new TemplateVariables(); From 6658203493dc80d2403657f98abf6c27d04a6685 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Mon, 30 Dec 2024 10:15:42 +0800 Subject: [PATCH 076/118] fix(integration): it doesn't sync internal schemas in the project in the bastion mode #4109 --- .../service/connection/database/DatabaseService.java | 10 +++++++--- .../odc/service/db/schema/DBSchemaSyncTaskManager.java | 8 +++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index cf78339e3c..f415778fd5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -46,6 +46,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; @@ -224,6 +225,9 @@ public class DatabaseService { @Autowired private ConnectionSyncHistoryService connectionSyncHistoryService; + @Value("${odc.integration.bastion.enabled:false}") + private boolean bastionEnabled; + @Transactional(rollbackFor = Exception.class) @SkipAuthorize("internal authenticated") public Database detail(@NonNull Long id) { @@ -545,7 +549,7 @@ public int updateEnvironmentIdByConnectionId(@NotNull Long environmentId, @NotNu private void syncTeamDataSources(ConnectionConfig connection) throws ExecutionException, InterruptedException, TimeoutException { Long currentProjectId = connection.getProjectId(); - boolean blockExcludeSchemas = dbSchemaSyncProperties.isBlockExclusionsWhenSyncDbToProject(); + boolean blockExcludeSchemas = dbSchemaSyncProperties.isBlockExclusionsWhenSyncDbToProject() && !bastionEnabled; List excludeSchemas = dbSchemaSyncProperties.getExcludeSchemas(connection.getDialectType()); DataSource teamDataSource = getDataSourceFactory(connection).getDataSource(); ExecutorService executorService = Executors.newFixedThreadPool(1); @@ -671,7 +675,7 @@ private Long getProjectId(DatabaseEntity database, Long currentProjectId, List databases, @N ErrorCodes.AccessDenied, null, "Lack of update permission on current datasource"); Map id2Conn = connectionService.innerListByIds(connectionIds).stream() .collect(Collectors.toMap(ConnectionConfig::getId, c -> c, (c1, c2) -> c2)); - if (dbSchemaSyncProperties.isBlockExclusionsWhenSyncDbToProject()) { + if (dbSchemaSyncProperties.isBlockExclusionsWhenSyncDbToProject() && !bastionEnabled) { connectionIds = databases.stream().filter(database -> { ConnectionConfig connection = id2Conn.get(database.getConnectionId()); return connection != null && !dbSchemaSyncProperties.getExcludeSchemas(connection.getDialectType()) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java index a7440a5cb0..c4d443a1d1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java @@ -27,6 +27,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Component; @@ -82,6 +83,10 @@ public class DBSchemaSyncTaskManager { @Autowired private GlobalSearchProperties globalSearchProperties; + @Value("${odc.integration.bastion.enabled:false}") + private boolean bastionEnabled; + + private final LoadingCache datasourceId2UserEntity = CacheBuilder.newBuilder().maximumSize(100) .expireAfterWrite(10, TimeUnit.MINUTES) .build(new CacheLoader() { @@ -115,7 +120,8 @@ public void submitTaskByDatabases(@NonNull Collection databases) { public void submitTaskByDataSource(@NonNull ConnectionConfig dataSource) { List databases = databaseService.listExistDatabasesByConnectionId(dataSource.getId()); databases.removeIf(e -> (syncProperties.isBlockExclusionsWhenSyncDbSchemas() - && syncProperties.getExcludeSchemas(dataSource.getDialectType()).contains(e.getName())) + && syncProperties.getExcludeSchemas(dataSource.getDialectType()).contains(e.getName()) + && !bastionEnabled) || e.getObjectSyncStatus() == DBObjectSyncStatus.PENDING); submitTaskByDatabases(databases); } From cc2a493d01585c222c22455f21ddc392bc3c1049 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Mon, 30 Dec 2024 15:25:34 +0800 Subject: [PATCH 077/118] fix(session):drop pl require database change permission #4112 --- .../session/util/DBSchemaExtractor.java | 89 +++++++++-------- .../session/test_db_schema_extractor.yaml | 95 ++++++++++++++++++- 2 files changed, 140 insertions(+), 44 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java index efb199298a..c3a9b69b60 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java @@ -149,7 +149,7 @@ private static Set listDBSchemas(AbstractSyntaxTree ast, Diale visitor.visit(ast.getRoot()); identities = visitor.getIdentities(); } else { - OBMySQLRelationFactorVisitor visitor = new OBMySQLRelationFactorVisitor(); + OBMySQLRelationFactorVisitor visitor = new OBMySQLRelationFactorVisitor(defaultSchema); visitor.visit(ast.getRoot()); identities = visitor.getIdentities(); } @@ -167,7 +167,7 @@ private static Set listDBSchemas(AbstractSyntaxTree ast, Diale visitor.visit(ast.getRoot()); identities = visitor.getIdentities(); } else { - OBOracleRelationFactorVisitor visitor = new OBOracleRelationFactorVisitor(); + OBOracleRelationFactorVisitor visitor = new OBOracleRelationFactorVisitor(defaultSchema); visitor.visit(ast.getRoot()); identities = visitor.getIdentities(); } @@ -200,15 +200,18 @@ private static class OBMySQLRelationFactorVisitor extends OBParserBaseVisitor identities = new HashSet<>(); + private final String defaultSchema; + + private OBMySQLRelationFactorVisitor(String defaultSchema) { + this.defaultSchema = defaultSchema; + } + @Override public RelationFactor visitDrop_function_stmt( com.oceanbase.tools.sqlparser.obmysql.OBParser.Drop_function_stmtContext ctx) { RelationFactor relationFactor = MySQLFromReferenceFactory.getRelationFactor( ctx.relation_factor().normal_relation_factor()); - if (relationFactor.getSchema() != null) { - identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); - } - return null; + return extractIdentityExceptTableAndView(relationFactor); } @Override @@ -216,10 +219,7 @@ public RelationFactor visitDrop_procedure_stmt( com.oceanbase.tools.sqlparser.obmysql.OBParser.Drop_procedure_stmtContext ctx) { RelationFactor relationFactor = MySQLFromReferenceFactory.getRelationFactor( ctx.relation_factor().normal_relation_factor()); - if (relationFactor.getSchema() != null) { - identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); - } - return null; + return extractIdentityExceptTableAndView(relationFactor); } @Override @@ -227,10 +227,7 @@ public RelationFactor visitDrop_trigger_stmt( com.oceanbase.tools.sqlparser.obmysql.OBParser.Drop_trigger_stmtContext ctx) { RelationFactor relationFactor = MySQLFromReferenceFactory.getRelationFactor( ctx.relation_factor().normal_relation_factor()); - if (relationFactor.getSchema() != null) { - identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); - } - return null; + return extractIdentityExceptTableAndView(relationFactor); } @Override @@ -322,6 +319,15 @@ private void addRelationFactor(RelationFactor rf) { identities.add(new DBSchemaIdentity(rf.getSchema(), rf.getRelation())); } } + + private RelationFactor extractIdentityExceptTableAndView(RelationFactor relationFactor) { + if (StringUtils.isNotBlank(relationFactor.getSchema())) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } else if (StringUtils.isNotBlank(defaultSchema)) { + identities.add(new DBSchemaIdentity(defaultSchema, null)); + } + return null; + } } @@ -386,65 +392,58 @@ private static class OBOracleRelationFactorVisitor private final Set identities = new HashSet<>(); + private final String defaultSchema; + + private OBOracleRelationFactorVisitor() { + this.defaultSchema = null; + } + + private OBOracleRelationFactorVisitor(String defaultSchema) { + this.defaultSchema = defaultSchema; + } + @Override public RelationFactor visitDrop_package_stmt(OBParser.Drop_package_stmtContext ctx) { RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); - if (relationFactor.getSchema() != null) { - identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); - } - return null; + return extractIdentityExceptTableAndView(relationFactor); } @Override public RelationFactor visitDrop_procedure_stmt(OBParser.Drop_procedure_stmtContext ctx) { RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); - if (relationFactor.getSchema() != null) { - identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); - } - return null; + return extractIdentityExceptTableAndView(relationFactor); } @Override public RelationFactor visitDrop_function_stmt(OBParser.Drop_function_stmtContext ctx) { RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); - if (relationFactor.getSchema() != null) { - identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); - } - return null; + return extractIdentityExceptTableAndView(relationFactor); } @Override public RelationFactor visitDrop_trigger_stmt(OBParser.Drop_trigger_stmtContext ctx) { RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); - if (relationFactor.getSchema() != null) { - identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); - } - return null; + return extractIdentityExceptTableAndView(relationFactor); } @Override public RelationFactor visitDrop_type_stmt(OBParser.Drop_type_stmtContext ctx) { RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); - if (relationFactor.getSchema() != null) { - identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); - } - return null; + return extractIdentityExceptTableAndView(relationFactor); } @Override public RelationFactor visitDrop_sequence_stmt(OBParser.Drop_sequence_stmtContext ctx) { RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); - if (relationFactor.getSchema() != null) { - identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); - } - return null; + return extractIdentityExceptTableAndView(relationFactor); } @Override public RelationFactor visitDrop_synonym_stmt(OBParser.Drop_synonym_stmtContext ctx) { - String databaseName = ctx.database_factor().getText(); - if (databaseName != null) { - identities.add(new DBSchemaIdentity(databaseName, null)); + if (Objects.nonNull(ctx.database_factor()) && StringUtils.isNotBlank(ctx.database_factor().getText())) { + identities.add(new DBSchemaIdentity(ctx.database_factor().getText(), null)); + } else if (StringUtils.isNotBlank(defaultSchema)) { + identities.add(new DBSchemaIdentity(defaultSchema, null)); } return null; } @@ -559,6 +558,14 @@ private void addRelationFactor(RelationFactor rf) { } } + private RelationFactor extractIdentityExceptTableAndView(RelationFactor relationFactor) { + if (StringUtils.isNotBlank(relationFactor.getSchema())) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } else if (StringUtils.isNotBlank(defaultSchema)) { + identities.add(new DBSchemaIdentity(defaultSchema, null)); + } + return null; + } } diff --git a/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml b/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml index e41c1d257a..5e69004838 100644 --- a/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml +++ b/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml @@ -184,7 +184,7 @@ sqls: - "SELECT * FROM DB1.TABLE1@FAKE_DBLINK;" expected: [ ] -# drop function ,procedure and trigger +# drop function ,procedure and trigger with dbname - id: 23 dialect_type: OB_MYSQL default_schema: DEFAULT_SCHEMA @@ -233,7 +233,7 @@ expected: - schema: DB1 table: ~ -#drop sequence、type、package、synonym、public synonym for ob oracle +#drop sequence、type、package、synonym、public synonym with dbname for ob oracle - id: 29 dialect_type: OB_ORACLE default_schema: DEFAULT_SCHEMA @@ -274,5 +274,94 @@ expected: - schema: DB1 table: ~ - +# drop function ,procedure and trigger without dbname +- id: 34 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop function func1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 35 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop procedure proc1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 36 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop trigger trigger1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 37 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP FUNCTION FUNC1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 38 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PROCEDURE PROC1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 39 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP TRIGGER TRIGGER1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +#drop sequence、type、package、synonym、public synonym without dbname for ob oracle +- id: 40 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP SEQUENCE SEQ1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 41 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP TYPE TYPE1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 42 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PACKAGE PACKAGE1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 43 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP SYNONYM SYNONYM1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id : 44 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PUBLIC SYNONYM SYNONYM1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ From 8796da202cb764f5cb556b2a416fb9feec8ba01c Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Tue, 31 Dec 2024 11:39:47 +0800 Subject: [PATCH 078/118] fix(db): The method of determining whether opening the global client session is incorrect * Refactor global client session check in DefaultDBSessionManage * Add log for failed column 'time' lookup in DefaultDBSessionManage --- .../db/session/DefaultDBSessionManage.java | 46 ++++++------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index 6f92c763a0..99c0bb22fa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -18,6 +18,7 @@ import static com.oceanbase.odc.core.shared.constant.DialectType.OB_MYSQL; import java.sql.Connection; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.HashSet; @@ -300,38 +301,19 @@ private boolean isGlobalClientSessionEnabled(ConnectionSession connectionSession || VersionUtils.isLessThan(obVersion, GLOBAL_CLIENT_SESSION_OB_VERSION_NUMBER)) { return false; } - try { - Integer proxyId = getOBProxyConfig(connectionSession, "proxy_id"); - Integer clientSessionIdVersion = getOBProxyConfig(connectionSession, "client_session_id_version"); - - return proxyId != null - && proxyId >= GLOBAL_CLIENT_SESSION_PROXY_ID_MIN - && proxyId <= GLOBAL_CLIENT_SESSION_PROXY_ID_MAX - && clientSessionIdVersion != null - && clientSessionIdVersion == GLOBAL_CLIENT_SESSION_ID_VERSION; - } catch (Exception e) { - log.warn("Failed to determine if global client session is enabled: {}", e.getMessage()); - return false; - } - } - - /** - * Gets the value of OBProxy's configuration variable If an exception occurs or the version does not - * support, return null. - * - * @param connectionSession - * @param configName - * @return - */ - private Integer getOBProxyConfig(ConnectionSession connectionSession, String configName) { - try { - return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) - .query("show proxyconfig like '" + configName + "';", - rs -> rs.next() ? rs.getInt("value") : null); - } catch (Exception e) { - log.warn("Failed to obtain the value of OBProxy's configuration variable: {}", e.getMessage()); - return null; - } + // Check whether the global session is open + // If the global session is open, the "time" column will be displayed in the result set after + // executing the sql statement of "show processlist" + return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) + .query("show processlist", rs -> { + try { + int columnIndex = rs.findColumn("time"); + return columnIndex > 0; + } catch (SQLException e) { + log.warn("Failed to find the column 'time' in the result set", e); + return false; + } + }); } private boolean isUnknownThreadIdError(Exception e) { From 4d5ec77533c52a4d28ad6c86a836e964881becb6 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Thu, 2 Jan 2025 10:15:33 +0800 Subject: [PATCH 079/118] fix(migrate): complete data for connect_database.connect_type (#4113) * complete connect_database.connect_type if is null * update sql --- .../migrate/common/V_4_3_3_6__complete_connect_database.sql | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_6__complete_connect_database.sql diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_6__complete_connect_database.sql b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_6__complete_connect_database.sql new file mode 100644 index 0000000000..6c3a76cb56 --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_6__complete_connect_database.sql @@ -0,0 +1,6 @@ +UPDATE connect_database +SET connect_type = ( + SELECT cc.type FROM connect_connection cc + WHERE connect_database.connection_id = cc.id + ) +WHERE connect_type IS NULL AND type = 'PHYSICAL'; \ No newline at end of file From 76d11f41e6472be1419928a434e87e2fe6d6f9cb Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Thu, 2 Jan 2025 11:12:26 +0800 Subject: [PATCH 080/118] fix(permission): global project role cannot see approvable tickets #4116 --- .../collaboration/project/ProjectService.java | 1 + .../collaboration/project/model/Project.java | 3 +++ .../odc/service/iam/PermissionService.java | 17 ----------------- .../odc/service/iam/ResourceRoleService.java | 17 +++++++++-------- .../oceanbase/odc/service/iam/UserService.java | 1 + 5 files changed, 14 insertions(+), 25 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java index cff5ab0a6c..a03711a8c5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java @@ -627,6 +627,7 @@ private ProjectMember fromUserResourceRole(UserResourceRole userResourceRole) { member.setName(user.getName()); member.setAccountName(user.getAccountName()); member.setUserEnabled(user.isEnabled()); + member.setDerivedFromGlobalProjectRole(userResourceRole.isDerivedFromGlobalProjectRole()); return member; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/Project.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/Project.java index 4551a9fc4c..98a2b9ede0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/Project.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/Project.java @@ -129,5 +129,8 @@ public static class ProjectMember { private String name; private ResourceRoleName role; + + @JsonProperty(access = Access.READ_ONLY) + private boolean derivedFromGlobalProjectRole; } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java index aba2e7c17d..681a1f06b1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java @@ -16,8 +16,6 @@ package com.oceanbase.odc.service.iam; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; @@ -122,19 +120,4 @@ public void deleteExpiredPermission(Date expiredTime) { log.info("Clear expired permission, count: {}, expired time: {}", count, expiredTime); } } - - @SkipAuthorize - public List findGlobalResourceRolePermissions(Long userId, Long organizationId) { - return findByUserIdAndOrganizationIdAndResourceIdentifierAndActionIn(userId, organizationId, - "ODC_PROJECT:*", Arrays.asList("OWNER", "DBA", "SECURITY_ADMINISTRATOR")); - } - - - - private List findByUserIdAndOrganizationIdAndResourceIdentifierAndActionIn(Long userId, - Long organizationId, String resourceIdentifier, Collection actions) { - return permissionRepository.findByUserIdAndOrganizationIdAndResourceIdentifierAndActionIn(userId, - organizationId, resourceIdentifier, actions); - } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java index b3df92fb61..52c5b946f6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java @@ -38,7 +38,6 @@ import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.UnexpectedException; import com.oceanbase.odc.metadb.collaboration.ProjectRepository; -import com.oceanbase.odc.metadb.iam.PermissionEntity; import com.oceanbase.odc.metadb.iam.resourcerole.ResourceRoleEntity; import com.oceanbase.odc.metadb.iam.resourcerole.ResourceRoleRepository; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; @@ -120,13 +119,14 @@ public List saveAll(List userResourceRoleLis public Set getResourceRoleIdentifiersByUserId(long organizationId, long userId) { List userResourceRoleEntities = userResourceRoleRepository.findByOrganizationIdAndUserId(organizationId, userId); - List globalResourceRolePermissions = - permissionService.findGlobalResourceRolePermissions(userId, organizationId); + List globalResourceRoles = + globalResourceRoleService.findGlobalResourceRoleUsersByOrganizationIdAndUserId(organizationId, userId) + .stream().map(UserGlobalResourceRole::getResourceRole).collect(Collectors.toList()); Set resourceRoleIdentifiers = userResourceRoleEntities.stream() .map(i -> StringUtils.join(i.getResourceId(), ":", i.getResourceRoleId())) .collect(Collectors.toSet()); - if (CollectionUtils.isEmpty(globalResourceRolePermissions)) { + if (CollectionUtils.isEmpty(globalResourceRoles)) { return resourceRoleIdentifiers; } // Has global resource role @@ -134,8 +134,8 @@ public Set getResourceRoleIdentifiersByUserId(long organizationId, long .stream().map(resourceRoleMapper::entityToModel) .collect(Collectors.toMap(role -> role.getRoleName().name(), ResourceRole::getId, (v1, v2) -> v2)); - Set derivedFromGlobalResourceRole = globalResourceRolePermissions.stream() - .map(i -> StringUtils.join("*", ":", resourceRoleName2Id.get(i.getAction()))) + Set derivedFromGlobalResourceRole = globalResourceRoles.stream() + .map(i -> StringUtils.join("*", ":", resourceRoleName2Id.get(i.name()))) .collect(Collectors.toSet()); derivedFromGlobalResourceRole.addAll(resourceRoleIdentifiers); return derivedFromGlobalResourceRole; @@ -157,8 +157,9 @@ public Map> getProjectId2ResourceRoleNames(Long user .collect(Collectors.groupingBy(UserResourceRoleEntity::getResourceId, Collectors.mapping( e -> id2ResourceRoles.get(e.getResourceRoleId()).getRoleName(), Collectors.toSet()))); Set globalResourceRoles = - permissionService.findGlobalResourceRolePermissions(userId, organizationId).stream() - .map(i -> ResourceRoleName.valueOf(i.getAction())).collect(Collectors.toSet()); + globalResourceRoleService.findGlobalResourceRoleUsersByOrganizationIdAndUserId(organizationId, userId) + .stream() + .map(UserGlobalResourceRole::getResourceRole).collect(Collectors.toSet()); if (CollectionUtils.isEmpty(globalResourceRoles)) { return result; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java index 9fc080e6f6..74d2285eb3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java @@ -603,6 +603,7 @@ public PaginatedData listUserBasicsWithoutPermissionCheck() { List users = userRepository.findByOrganizationId(authenticationFacade.currentOrganizationId()).stream() .map(User::new).collect(Collectors.toList()); + acquireRolesAndRoleIds(users); return new PaginatedData<>(users, CustomPage.empty()); } From a1686991712eab180ffee3c2ea0d1d95bb829988 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Thu, 2 Jan 2025 15:01:54 +0800 Subject: [PATCH 081/118] fix(db):miss quote of pl name which contains @ causes pl edit failed #4115 --- .../com/oceanbase/odc/service/db/DBPLModifyHelper.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java index 24b1325e39..a3e193d503 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java @@ -48,6 +48,7 @@ import com.oceanbase.odc.service.session.model.SqlAsyncExecuteResp; import com.oceanbase.odc.service.session.model.SqlExecuteResult; import com.oceanbase.tools.dbbrowser.model.DBObjectType; +import com.oceanbase.tools.dbbrowser.util.MySQLSqlBuilder; import lombok.NonNull; @@ -103,14 +104,14 @@ private EditPLResp executeWrappedEditPL(String sessionId, EditPLReq editPLReq, String escapeRegexPlName = StringUtils.escapeRegex(plName) .orElseThrow(() -> new IllegalStateException(String.format("%s name cannot be null", plType))); String tempPLSql = editPLSql.replaceFirst(escapeRegexPlName, tempPlName); - StringBuilder wrappedSqlBuilder = new StringBuilder(); + MySQLSqlBuilder wrappedSqlBuilder = new MySQLSqlBuilder(); ConnectionSession connectionSession = sessionService.nullSafeGet(sessionId, true); SqlCommentProcessor processor = ConnectionSessionUtil.getSqlCommentProcessor(connectionSession); String delimiter = processor.getDelimiter(); wrappedSqlBuilder.append("DELIMITER $$\n") .append(tempPLSql).append(" $$\n") - .append("drop ").append(plType).append(" if exists ").append(tempPlName).append(" $$\n") - .append("drop ").append(plType).append(" if exists ").append(plName).append(" $$\n") + .append("drop ").append(plType).append(" if exists ").identifier(tempPlName).append(" $$\n") + .append("drop ").append(plType).append(" if exists ").identifier(plName).append(" $$\n") .append(editPLSql).append(" $$\n") .append("DELIMITER " + delimiter); String wrappedSql = wrappedSqlBuilder.toString(); @@ -118,7 +119,6 @@ private EditPLResp executeWrappedEditPL(String sessionId, EditPLReq editPLReq, sqlAsyncExecuteReq.setSql(wrappedSql); sqlAsyncExecuteReq.setSplit(true); sqlAsyncExecuteReq.setContinueExecutionOnError(false); - Lock editPLLock = obtainEditPLLock(connectionSession, plType); if (!editPLLock.tryLock(LOCK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { throw new ConflictException(ErrorCodes.ResourceModifying, "Can not acquire jdbc lock"); From e6e06abeabbb31897055ee43f868dc4c2701b4e6 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Thu, 2 Jan 2025 15:36:19 +0800 Subject: [PATCH 082/118] fix(function):The function does not display properly if the return value type is year (#4093) * Add YEAR support in OBMysqlCallFunctionCallBackTest and JdbcDataTypeUtil * Enhance YEAR type handling in OBMysqlCallFunctionCallBackTest and JdbcDataTypeUtil * Refactor OBMysqlCallFunctionCallBack and move tests * Optimize error handling in OBMysqlCallFunctionCallBack * Optimize row handling and import CollectionUtils in OBMysqlCallFunctionCallBack * Refactor OBMysqlCallFunctionCallBack, optimize imports and error handling * Add logging for function call failures in OBMysqlCallFunctionCallBack --- .../OBMysqlCallFunctionCallBackTest.java | 88 ++++++++++++++++--- .../db/mysql_function_callback_test.sql | 13 ++- .../oceanbase/odc/service/db/DBPLService.java | 4 +- .../db/util/OBMysqlCallFunctionCallBack.java | 45 +++++++--- 4 files changed, 124 insertions(+), 26 deletions(-) rename server/{odc-service/src/test/java/com/oceanbase/odc/service/db/util => integration-test/src/test/java/com/oceanbase/odc/service/session}/OBMysqlCallFunctionCallBackTest.java (67%) rename server/{odc-service => integration-test}/src/test/resources/db/mysql_function_callback_test.sql (54%) diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBackTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/session/OBMysqlCallFunctionCallBackTest.java similarity index 67% rename from server/odc-service/src/test/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBackTest.java rename to server/integration-test/src/test/java/com/oceanbase/odc/service/session/OBMysqlCallFunctionCallBackTest.java index d89d190ff2..1a1dd301df 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBackTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/session/OBMysqlCallFunctionCallBackTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.db.util; +package com.oceanbase.odc.service.session; import java.io.IOException; import java.io.InputStream; @@ -30,19 +30,33 @@ import org.springframework.jdbc.core.ConnectionCallback; import org.springframework.jdbc.core.JdbcTemplate; +import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.TestConnectionUtil; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.core.sql.execute.mapper.DefaultJdbcRowMapper; import com.oceanbase.odc.service.db.model.CallFunctionReq; import com.oceanbase.odc.service.db.model.CallFunctionResp; import com.oceanbase.odc.service.db.model.PLOutParam; +import com.oceanbase.odc.service.db.util.OBMysqlCallFunctionCallBack; import com.oceanbase.odc.test.database.TestDBConfigurations; import com.oceanbase.tools.dbbrowser.model.DBFunction; import com.oceanbase.tools.dbbrowser.model.DBPLParam; import com.oceanbase.tools.dbbrowser.model.DBPLParamMode; -public class OBMysqlCallFunctionCallBackTest { +/** + * @description: + * @author: zijia.cj + * @date: 2024/12/30 15:52 + * @since: 4.3.3 + */ +public class OBMysqlCallFunctionCallBackTest extends ServiceTestEnv { + + public static final String TEST_CASE_1 = "TEST_CASE_1"; + public static final String TEST_CASE_2 = "TEST_CASE_2"; + public static final String TEST_CASE_3 = "TEST_CASE_3"; + public static final String TEST_CASE_4 = "func_test_4"; - public static final String TEST_CASE_1 = "func_test"; - public static final String TEST_CASE_2 = "func_test_1"; - public static final String TEST_CASE_3 = "func_test_2"; @BeforeClass public static void setUp() throws IOException { @@ -58,6 +72,7 @@ public static void clear() { mysql.execute("DROP FUNCTION " + TEST_CASE_1); mysql.execute("DROP FUNCTION " + TEST_CASE_2); mysql.execute("DROP FUNCTION " + TEST_CASE_3); + mysql.execute("DROP FUNCTION " + TEST_CASE_4); } @Test @@ -83,7 +98,10 @@ public void doInConnection_normalFunction_callSucceed() { function.setReturnType("int"); callFunctionReq.setFunction(function); - ConnectionCallback callback = new OBMysqlCallFunctionCallBack(callFunctionReq, -1); + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + ConnectionCallback callback = + new OBMysqlCallFunctionCallBack(callFunctionReq, -1, defaultJdbcRowMapper); JdbcTemplate jdbcTemplate = new JdbcTemplate(TestDBConfigurations.getInstance().getTestOBMysqlConfiguration().getDataSource()); CallFunctionResp actual = jdbcTemplate.execute(callback); @@ -128,7 +146,10 @@ public void doInConnection_nullParamExists_callSucceed() { function.setReturnType("int"); callFunctionReq.setFunction(function); - ConnectionCallback callback = new OBMysqlCallFunctionCallBack(callFunctionReq, -1); + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + ConnectionCallback callback = + new OBMysqlCallFunctionCallBack(callFunctionReq, -1, defaultJdbcRowMapper); JdbcTemplate jdbcTemplate = new JdbcTemplate(TestDBConfigurations.getInstance().getTestOBMysqlConfiguration().getDataSource()); CallFunctionResp actual = jdbcTemplate.execute(callback); @@ -173,7 +194,10 @@ public void doInConnection_emptyParamExists_callSucceed() { function.setReturnType("int"); callFunctionReq.setFunction(function); - ConnectionCallback callback = new OBMysqlCallFunctionCallBack(callFunctionReq, -1); + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + ConnectionCallback callback = + new OBMysqlCallFunctionCallBack(callFunctionReq, -1, defaultJdbcRowMapper); JdbcTemplate jdbcTemplate = new JdbcTemplate(TestDBConfigurations.getInstance().getTestOBMysqlConfiguration().getDataSource()); CallFunctionResp actual = jdbcTemplate.execute(callback); @@ -205,7 +229,10 @@ public void doInConnection_unusualParam_callSucceed() { function.setParams(list); callFunctionReq.setFunction(function); - ConnectionCallback callback = new OBMysqlCallFunctionCallBack(callFunctionReq, -1); + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + ConnectionCallback callback = + new OBMysqlCallFunctionCallBack(callFunctionReq, -1, defaultJdbcRowMapper); JdbcTemplate jdbcTemplate = new JdbcTemplate(TestDBConfigurations.getInstance().getTestOBMysqlConfiguration().getDataSource()); CallFunctionResp actual = jdbcTemplate.execute(callback); @@ -220,6 +247,48 @@ public void doInConnection_unusualParam_callSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void doInConnection_returnTypeIsYear_callSucceed() { + testCallFunctionWhenReturnIsYear("2024", "2024"); + testCallFunctionWhenReturnIsYear("0000", "0000"); + testCallFunctionWhenReturnIsYear("0", "2000"); + testCallFunctionWhenReturnIsYear("1", "2001"); + testCallFunctionWhenReturnIsYear("99", "1999"); + } + + private static void testCallFunctionWhenReturnIsYear(String input, String expectOutput) { + CallFunctionReq callFunctionReq = new CallFunctionReq(); + DBFunction function = new DBFunction(); + function.setFunName(TEST_CASE_4); + List list = new ArrayList<>(); + DBPLParam param = new DBPLParam(); + param.setParamName("p1"); + param.setDefaultValue(input); + param.setDataType("year"); + param.setParamMode(DBPLParamMode.IN); + list.add(param); + function.setParams(list); + function.setReturnType("year"); + callFunctionReq.setFunction(function); + + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + ConnectionCallback callback = + new OBMysqlCallFunctionCallBack(callFunctionReq, -1, defaultJdbcRowMapper); + JdbcTemplate jdbcTemplate = + new JdbcTemplate(TestDBConfigurations.getInstance().getTestOBMysqlConfiguration().getDataSource()); + CallFunctionResp actual = jdbcTemplate.execute(callback); + + CallFunctionResp expect = new CallFunctionResp(); + PLOutParam plOutParam = new PLOutParam(); + plOutParam.setParamName(TEST_CASE_4); + plOutParam.setDataType("year"); + plOutParam.setValue(expectOutput); + expect.setReturnValue(plOutParam); + expect.setOutParams(null); + Assert.assertEquals(expect, actual); + } + private static List getContent() throws IOException { String delimiter = "\\$\\$\\s*"; try (InputStream input = OBMysqlCallFunctionCallBackTest.class.getClassLoader() @@ -230,5 +299,4 @@ private static List getContent() throws IOException { return new ArrayList<>(Arrays.asList(substitutor.replace(new String(buffer)).split(delimiter))); } } - } diff --git a/server/odc-service/src/test/resources/db/mysql_function_callback_test.sql b/server/integration-test/src/test/resources/db/mysql_function_callback_test.sql similarity index 54% rename from server/odc-service/src/test/resources/db/mysql_function_callback_test.sql rename to server/integration-test/src/test/resources/db/mysql_function_callback_test.sql index 975f9b4d89..cbe187af19 100644 --- a/server/odc-service/src/test/resources/db/mysql_function_callback_test.sql +++ b/server/integration-test/src/test/resources/db/mysql_function_callback_test.sql @@ -1,4 +1,4 @@ -create function ${const:com.oceanbase.odc.service.db.util.OBMysqlCallFunctionCallBackTest.TEST_CASE_1} ( +create function ${const:com.oceanbase.odc.service.session.OBMysqlCallFunctionCallBackTest.TEST_CASE_1} ( p1 int, p2 int) returns int begin @@ -6,7 +6,7 @@ begin end; $$ -create function ${const:com.oceanbase.odc.service.db.util.OBMysqlCallFunctionCallBackTest.TEST_CASE_2} ( +create function ${const:com.oceanbase.odc.service.session.OBMysqlCallFunctionCallBackTest.TEST_CASE_2} ( p0 int, p1 int, p2 varchar(20)) returns int @@ -21,9 +21,16 @@ begin end; $$ -create function ${const:com.oceanbase.odc.service.db.util.OBMysqlCallFunctionCallBackTest.TEST_CASE_3} ( +create function ${const:com.oceanbase.odc.service.session.OBMysqlCallFunctionCallBackTest.TEST_CASE_3} ( p0 varchar(20)) returns varchar(20) begin return p0; end; +$$ + +create function ${const:com.oceanbase.odc.service.session.OBMysqlCallFunctionCallBackTest.TEST_CASE_4} ( +p1 year) returns year +begin +return p1; +end; $$ \ No newline at end of file diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java index 40d3570b96..ae23a61aeb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java @@ -51,6 +51,7 @@ import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.core.shared.exception.UnsupportedException; import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor; +import com.oceanbase.odc.core.sql.execute.mapper.DefaultJdbcRowMapper; import com.oceanbase.odc.core.sql.execute.task.DefaultSqlExecuteTaskManager; import com.oceanbase.odc.core.sql.split.OffsetString; import com.oceanbase.odc.core.sql.split.SqlCommentProcessor; @@ -287,7 +288,8 @@ public String callFunction(@NonNull ConnectionSession session, @NonNull CallFunc if (dialectType.isOracle()) { callback = new OBOracleCallFunctionBlockCallBack(req, -1); } else if (dialectType.isMysql()) { - callback = new OBMysqlCallFunctionCallBack(req, -1); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + callback = new OBMysqlCallFunctionCallBack(req, -1, defaultJdbcRowMapper); } else { throw new IllegalArgumentException("Illegal dialect type, " + dialectType); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBack.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBack.java index 51219453ad..84914deed9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBack.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBack.java @@ -24,12 +24,14 @@ import java.util.Objects; import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.Validate; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.ConnectionCallback; import com.oceanbase.odc.common.util.StringUtils; -import com.oceanbase.odc.core.sql.util.JdbcDataTypeUtil; +import com.oceanbase.odc.core.sql.execute.mapper.JdbcRowMapper; +import com.oceanbase.odc.core.sql.execute.model.JdbcQueryResult; import com.oceanbase.odc.service.db.model.CallFunctionReq; import com.oceanbase.odc.service.db.model.CallFunctionResp; import com.oceanbase.odc.service.db.model.PLOutParam; @@ -39,21 +41,25 @@ import com.oceanbase.tools.dbbrowser.util.SqlBuilder; import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class OBMysqlCallFunctionCallBack implements ConnectionCallback { private final DBFunction function; private final int timeoutSeconds; + private final JdbcRowMapper rowDataMapper; - public OBMysqlCallFunctionCallBack(@NonNull CallFunctionReq callFunctionReq, int timeoutSeconds) { + public OBMysqlCallFunctionCallBack(@NonNull CallFunctionReq callFunctionReq, int timeoutSeconds, + @NonNull JdbcRowMapper rowDataMapper) { Validate.notBlank(callFunctionReq.getFunction().getFunName(), "Function name can not be blank"); this.function = callFunctionReq.getFunction(); this.timeoutSeconds = timeoutSeconds; + this.rowDataMapper = rowDataMapper; } @Override public CallFunctionResp doInConnection(Connection con) throws SQLException, DataAccessException { - CallFunctionResp callFunctionResp = new CallFunctionResp(); List params = new ArrayList<>(); if (function.getParams() != null) { params = function.getParams(); @@ -76,23 +82,38 @@ public CallFunctionResp doInConnection(Connection con) throws SQLException, Data if (this.timeoutSeconds > 0) { stmt.setQueryTimeout(this.timeoutSeconds); } - PLOutParam plOutParam = new PLOutParam(); try (ResultSet res = stmt.executeQuery(sqlBuilder.toString())) { if (!res.next()) { - plOutParam.setValue(function.getReturnValue()); + return generateDefaultReturnValue(); + } + JdbcQueryResult jdbcQueryResult = new JdbcQueryResult(res.getMetaData(), rowDataMapper); + jdbcQueryResult.addLine(res); + List> rows = jdbcQueryResult.getRows(); + if (CollectionUtils.size(rows) == 1 && CollectionUtils.size(rows.get(0)) == 1) { + CallFunctionResp callFunctionResp = new CallFunctionResp(); + PLOutParam plOutParam = new PLOutParam(); + plOutParam.setValue(String.valueOf(rows.get(0).get(0))); plOutParam.setDataType(function.getReturnType()); callFunctionResp.setReturnValue(plOutParam); + callFunctionResp.setOutParams(null); return callFunctionResp; } - Object value = JdbcDataTypeUtil.getValueFromResultSet( - res, 1, function.getReturnType()); - plOutParam.setValue(String.valueOf(value)); - plOutParam.setDataType(function.getReturnType()); - callFunctionResp.setReturnValue(plOutParam); + throw new IllegalStateException("The return value of a function must be unique"); } + } catch (Exception e) { + log.warn("Failed to call function {}", function.getFunName(), e); + CallFunctionResp callFunctionResp = generateDefaultReturnValue(); + callFunctionResp.setErrorMessage(e.getMessage()); + return callFunctionResp; } - callFunctionResp.setOutParams(null); - return callFunctionResp; } + private CallFunctionResp generateDefaultReturnValue() { + CallFunctionResp callFunctionResp = new CallFunctionResp(); + PLOutParam plOutParam = new PLOutParam(); + plOutParam.setValue(function.getReturnValue()); + plOutParam.setDataType(function.getReturnType()); + callFunctionResp.setReturnValue(plOutParam); + return callFunctionResp; + } } From 7051c3443fdba1a71ce3831ffc3af6853e298c08 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Fri, 3 Jan 2025 10:09:54 +0800 Subject: [PATCH 083/118] fix(permission): global project roles cannot view/execute/approve tickets #4117 --- .../odc/config/BaseAuthConfiguration.java | 9 +- .../odc/config/DefaultAuthConfiguration.java | 6 +- .../odc/metadb/iam/PermissionRepository.java | 9 -- .../flow/ApprovalPermissionService.java | 8 +- .../odc/service/flow/FlowInstanceService.java | 2 +- .../service/flow/FlowPermissionHelper.java | 16 +++- .../service/flow/FlowTaskInstanceService.java | 2 +- .../factory/FlowResponseMapperFactory.java | 28 +++++-- .../ApprovalStatusNotifyListener.java | 14 ++-- .../iam/GlobalResourceRoleService.java | 11 +++ .../odc/service/iam/ResourceRoleService.java | 84 +++++++++++++++++-- .../iam/auth/DefaultAuthorizationFacade.java | 8 +- .../iam/auth/ResourceRoleAuthorizer.java | 12 +-- .../service/iam/model/UserResourceRole.java | 2 + .../iam/util/GlobalResourceRoleUtil.java | 6 ++ 15 files changed, 161 insertions(+), 56 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/config/BaseAuthConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/config/BaseAuthConfiguration.java index 396dfb777f..5f4edb1899 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/config/BaseAuthConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/config/BaseAuthConfiguration.java @@ -34,9 +34,9 @@ import com.oceanbase.odc.core.authority.permission.PermissionProvider; import com.oceanbase.odc.core.authority.session.factory.DefaultSecuritySessionFactory; import com.oceanbase.odc.metadb.iam.PermissionRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.iam.ResourcePermissionExtractor; import com.oceanbase.odc.service.iam.ResourceRoleBasedPermissionExtractor; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.iam.auth.DefaultPermissionProvider; import com.oceanbase.odc.service.iam.auth.EmptyAuthenticator; @@ -55,10 +55,10 @@ public abstract class BaseAuthConfiguration { @Bean public SecurityManager servletSecurityManager(PermissionRepository permissionRepository, - ResourcePermissionExtractor permissionMapper, UserResourceRoleRepository resourceRoleRepository, + ResourcePermissionExtractor permissionMapper, ResourceRoleService resourceRoleService, ResourceRoleBasedPermissionExtractor resourceRoleBasedPermissionExtractor, AuthenticationFacade authenticationFacade) { - Collection authorizers = authorizers(permissionRepository, permissionMapper, resourceRoleRepository, + Collection authorizers = authorizers(permissionRepository, permissionMapper, resourceRoleService, resourceRoleBasedPermissionExtractor); DefaultAuthorizerManager authorizerManager = new DefaultAuthorizerManager(authorizers); PermissionStrategy permissionStrategy = permissionStrategy(); @@ -94,7 +94,6 @@ protected ReturnValueProvider returnValueProvider(AuthorizerManager manager, Per } protected abstract Collection authorizers(PermissionRepository permissionRepository, - ResourcePermissionExtractor resourcePermissionExtractor, UserResourceRoleRepository resourceRoleRepository, + ResourcePermissionExtractor resourcePermissionExtractor, ResourceRoleService resourceRoleService, ResourceRoleBasedPermissionExtractor resourceRoleBasedPermissionExtractor); - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java index ef884b1598..f9e14e3d8a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java @@ -20,9 +20,9 @@ import com.oceanbase.odc.core.authority.auth.Authorizer; import com.oceanbase.odc.metadb.iam.PermissionRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.iam.ResourcePermissionExtractor; import com.oceanbase.odc.service.iam.ResourceRoleBasedPermissionExtractor; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.auth.ComposedAuthorizer; import com.oceanbase.odc.service.iam.auth.DefaultAuthorizer; import com.oceanbase.odc.service.iam.auth.ResourceRoleAuthorizer; @@ -39,11 +39,11 @@ public class DefaultAuthConfiguration extends BaseAuthConfiguration { @Override protected Collection authorizers(PermissionRepository permissionRepository, - ResourcePermissionExtractor resourcePermissionExtractor, UserResourceRoleRepository resourceRoleRepository, + ResourcePermissionExtractor resourcePermissionExtractor, ResourceRoleService resourceRoleService, ResourceRoleBasedPermissionExtractor resourceRoleBasedPermissionExtractor) { DefaultAuthorizer defaultAuthorizer = new DefaultAuthorizer(permissionRepository, resourcePermissionExtractor); ResourceRoleAuthorizer resourceRoleAuthorizer = - new ResourceRoleAuthorizer(resourceRoleRepository, resourceRoleBasedPermissionExtractor); + new ResourceRoleAuthorizer(resourceRoleService, resourceRoleBasedPermissionExtractor); ComposedAuthorizer composedAuthorizer = new ComposedAuthorizer(defaultAuthorizer, resourceRoleAuthorizer); return Arrays.asList(defaultAuthorizer, resourceRoleAuthorizer, composedAuthorizer); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java index 7b52bce20f..c2a0856ec7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/PermissionRepository.java @@ -97,15 +97,6 @@ Optional findByOrganizationIdAndActionAndResourceIdentifier(Lo nativeQuery = true) List findByExpireTimeBefore(@Param("expireTime") Date expireTime); - - @Query(value = "select p.* from iam_user_permission up inner join iam_permission p on up.permission_id=p.id " - + "where up.user_id=:userId and up.organization_id=:organizationId and p.resource_identifier=:resourceIdentifier " - + "and p.action in (:actions)", - nativeQuery = true) - List findByUserIdAndOrganizationIdAndResourceIdentifierAndActionIn(@Param("userId") Long userId, - @Param("organizationId") Long organizationId, @Param("resourceIdentifier") String resourceIdentifier, - @Param("actions") Collection actions); - @Modifying @Transactional @Query(value = "delete from iam_permission p where p.id in (:ids)", nativeQuery = true) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/ApprovalPermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/ApprovalPermissionService.java index 070c3bb365..b173078eed 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/ApprovalPermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/ApprovalPermissionService.java @@ -39,12 +39,12 @@ import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.metadb.iam.UserRepository; import com.oceanbase.odc.metadb.iam.UserRoleRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.flow.model.FlowNodeStatus; import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.UserService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.iam.model.UserResourceRole; import lombok.NonNull; @@ -163,12 +163,12 @@ private Map> getUsersByFlowInstanceIdsAndStatus(@NonNull C if (resourceRoleIdentifiers.isEmpty()) { return new HashMap<>(); } - Map> identifier2UserIds = userResourceRoleRepository.findByResourceIdsAndResourceRoleIdsIn( - resourceRoleIdentifiers).stream() + Map> identifier2UserIds = resourceRoleService + .listByResourceIdentifierIn(resourceRoleIdentifiers).stream() .collect(Collectors.groupingBy(o -> String.format("%s:%s", o.getResourceId(), o.getResourceRoleId()))) .entrySet() .stream().collect(Collectors.toMap(entry -> entry.getKey(), - entry -> entry.getValue().stream().map(UserResourceRoleEntity::getUserId).collect( + entry -> entry.getValue().stream().map(UserResourceRole::getUserId).collect( Collectors.toSet()))); // map approval instance ids to users ids Map> instanceId2UserIds = diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java index 9328db7877..76620d1853 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java @@ -727,7 +727,7 @@ public T mapFlowInstanceWithReadPermission(@NonNull Long flowInstanceId, Fun } public T mapFlowInstanceWithWritePermission(@NonNull Long flowInstanceId, Function mapper) { - return mapFlowInstance(flowInstanceId, mapper, flowPermissionHelper.withProjectOwnerCheck()); + return mapFlowInstance(flowInstanceId, mapper, flowPermissionHelper.withExecutableCheck()); } public T mapFlowInstanceWithApprovalPermission(@NonNull Long flowInstanceId, Function mapper) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowPermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowPermissionHelper.java index f11eaa53ce..f4a7c1b58d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowPermissionHelper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowPermissionHelper.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.service.flow; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Set; @@ -62,11 +62,11 @@ public Consumer withProjectMemberCheck() { .hasProjectRole(flowInstance.getProjectId(), ResourceRoleName.all())); } - public Consumer withProjectOwnerCheck() { + public Consumer withProjectOwnerOrDBACheck() { return withProjectPermissionCheck( flowInstance -> flowInstance.getProjectId() != null && projectPermissionValidator .hasProjectRole(flowInstance.getProjectId(), - Collections.singletonList(ResourceRoleName.OWNER))); + Arrays.asList(ResourceRoleName.OWNER, ResourceRoleName.DBA))); } public Consumer withApprovableCheck() { @@ -80,6 +80,16 @@ public Consumer withApprovableCheck() { }; } + public Consumer withExecutableCheck() { + return flowInstance -> { + try { + withCreatorCheck().accept(flowInstance); + } catch (Exception ex) { + withProjectOwnerOrDBACheck().accept(flowInstance); + } + }; + } + public Consumer withCreatorCheck() { return flowInstance -> { if (!Objects.equals(authenticationFacade.currentUserId(), flowInstance.getCreatorId())) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java index b4b69d7ff4..617e28a50e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java @@ -174,7 +174,7 @@ public class FlowTaskInstanceService { public FlowInstanceDetailResp executeTask(@NotNull Long id) throws IOException { List instances = filterTaskInstance(id, instance -> instance.getStatus() == FlowNodeStatus.PENDING, - flowPermissionHelper.withCreatorCheck()); + flowPermissionHelper.withExecutableCheck()); PreConditions.validExists(ResourceType.ODC_FLOW_TASK_INSTANCE, "flowInstanceId", id, () -> instances.size() > 0); Verify.singleton(instances, "FlowTaskInstance"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java index 2166f8f0b9..55f8a3a72e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java @@ -61,8 +61,6 @@ import com.oceanbase.odc.metadb.iam.UserRepository; import com.oceanbase.odc.metadb.iam.UserRoleEntity; import com.oceanbase.odc.metadb.iam.UserRoleRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.metadb.integration.IntegrationEntity; import com.oceanbase.odc.metadb.regulation.risklevel.RiskLevelRepository; import com.oceanbase.odc.metadb.task.TaskEntity; @@ -74,6 +72,7 @@ import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.connection.util.ConnectionMapper; +import com.oceanbase.odc.service.databasechange.model.DatabaseChangeDatabase; import com.oceanbase.odc.service.flow.ApprovalPermissionService; import com.oceanbase.odc.service.flow.instance.FlowInstance; import com.oceanbase.odc.service.flow.model.FlowInstanceDetailResp; @@ -83,7 +82,10 @@ import com.oceanbase.odc.service.flow.model.FlowNodeStatus; import com.oceanbase.odc.service.flow.model.FlowTaskExecutionStrategy; import com.oceanbase.odc.service.flow.task.model.DBStructureComparisonParameter; +import com.oceanbase.odc.service.flow.task.model.MultipleDatabaseChangeParameters; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.iam.model.UserResourceRole; import com.oceanbase.odc.service.integration.IntegrationService; import com.oceanbase.odc.service.integration.client.ApprovalClient; import com.oceanbase.odc.service.integration.model.ApprovalProperties; @@ -134,7 +136,7 @@ public class FlowResponseMapperFactory { @Autowired private FlowInstanceRepository flowInstanceRepository; @Autowired - private UserResourceRoleRepository userResourceRoleRepository; + private ResourceRoleService resourceRoleService; @Autowired private RiskLevelRepository riskLevelRepository; @Autowired @@ -226,9 +228,9 @@ private FlowNodeInstanceMapper generateNodeMapper(@NonNull Collection flow && candidateResourceRoleIdentifiers.isEmpty()) { return Collections.emptyList(); } else if (!candidateResourceRoleIdentifiers.isEmpty()) { - Set resourceRoleUserIds = userResourceRoleRepository - .findByResourceIdsAndResourceRoleIdsIn(candidateResourceRoleIdentifiers) - .stream().map(UserResourceRoleEntity::getUserId).collect(Collectors.toSet()); + Set resourceRoleUserIds = + resourceRoleService.listByResourceIdentifierIn(candidateResourceRoleIdentifiers) + .stream().map(UserResourceRole::getUserId).collect(Collectors.toSet()); return CollectionUtils.isEmpty(resourceRoleUserIds) ? Collections.emptyList() : userRepository.findByUserIdsAndEnabled(resourceRoleUserIds, true); } else if (candidateUserIds.isEmpty()) { @@ -329,6 +331,7 @@ private FlowInstanceMapper generateMapper(@NonNull Collection flowInstance .map(TaskEntity::getDatabaseId) .filter(Objects::nonNull).collect(Collectors.toSet()); + databaseIds.addAll(collectMultiDatabaseChangeDatabaseIds(taskId2TaskEntity)); databaseIds.addAll(collectDBStructureComparisonDatabaseIds(taskId2TaskEntity)); Set projectIds = new HashSet<>(); Map id2Project = new HashMap<>(); @@ -480,4 +483,17 @@ private Set collectApplyProjectIds(Map taskId2TaskEntity .collect(Collectors.toSet()); return applyProjectIds; } + + private Set collectMultiDatabaseChangeDatabaseIds(Map taskId2TaskEntity) { + Set databaseIds = new HashSet<>(); + taskId2TaskEntity.values().stream() + .filter(task -> task.getTaskType().equals(TaskType.MULTIPLE_ASYNC)) + .forEach(task -> { + MultipleDatabaseChangeParameters parameter = JsonUtils.fromJson( + task.getParametersJson(), MultipleDatabaseChangeParameters.class); + databaseIds.addAll(parameter.getDatabases().stream().map(DatabaseChangeDatabase::getId) + .collect(Collectors.toSet())); + }); + return databaseIds; + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/listener/ApprovalStatusNotifyListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/listener/ApprovalStatusNotifyListener.java index dc63c66be1..74cfc729b6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/listener/ApprovalStatusNotifyListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/listener/ApprovalStatusNotifyListener.java @@ -28,12 +28,12 @@ import com.oceanbase.odc.core.flow.util.EmptyExecutionListener; import com.oceanbase.odc.metadb.flow.FlowInstanceApprovalViewEntity; import com.oceanbase.odc.metadb.flow.FlowInstanceApprovalViewRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.service.flow.FlowInstanceService; import com.oceanbase.odc.service.flow.FlowableAdaptor; import com.oceanbase.odc.service.flow.instance.FlowApprovalInstance; +import com.oceanbase.odc.service.iam.ResourceRoleService; +import com.oceanbase.odc.service.iam.model.UserResourceRole; import com.oceanbase.odc.service.notification.Broker; import com.oceanbase.odc.service.notification.NotificationProperties; import com.oceanbase.odc.service.notification.helper.EventBuilder; @@ -59,9 +59,9 @@ public class ApprovalStatusNotifyListener extends EmptyExecutionListener { @Autowired private FlowableAdaptor flowableAdaptor; @Autowired - FlowInstanceApprovalViewRepository flowInstanceApprovalViewRepository; + private FlowInstanceApprovalViewRepository flowInstanceApprovalViewRepository; @Autowired - UserResourceRoleRepository userResourceRoleRepository; + private ResourceRoleService resourceRoleService; @Override protected void onExecutiuonStart(DelegateExecution execution) { @@ -79,12 +79,12 @@ protected void onExecutiuonStart(DelegateExecution execution) { List approvals = flowInstanceApprovalViewRepository.findByIdIn(Collections.singletonList(target.getId())); if (CollectionUtils.isNotEmpty(approvals)) { - List userResourceRoles = - userResourceRoleRepository.findByResourceIdsAndResourceRoleIdsIn( + List userResourceRoles = + resourceRoleService.listByResourceIdentifierIn( approvals.stream().map(FlowInstanceApprovalViewEntity::getResourceRoleIdentifier) .collect(Collectors.toSet())); approverIds = CollectionUtils.isEmpty(userResourceRoles) ? null - : userResourceRoles.stream().map(UserResourceRoleEntity::getUserId).collect(Collectors.toSet()); + : userResourceRoles.stream().map(UserResourceRole::getUserId).collect(Collectors.toSet()); } Event event = eventBuilder.ofPendingApprovalTask(taskEntity, approverIds); broker.enqueueEvent(event); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java index 483167fdf8..b2be8729b1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java @@ -18,9 +18,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; import com.oceanbase.odc.core.shared.constant.ResourceType; @@ -62,4 +64,13 @@ public List findGlobalResourceRoleUsersByOrganizationIdA return userRoleRepository.findByOrganizationIdAndNameIn( organizationId, Arrays.asList(GlobalResourceRoleUtil.getGlobalRoleName(resourceRoleName))); } + + public List findGlobalResourceRoleUsersByOrganizationIdAndRoleIn(Long organizationId, + Set resourceRoleNames) { + if (CollectionUtils.isEmpty(resourceRoleNames)) { + return Collections.emptyList(); + } + return userRoleRepository.findByOrganizationIdAndNameIn(organizationId, + GlobalResourceRoleUtil.getGlobalRoleName(resourceRoleNames)); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java index 52c5b946f6..178ada63d8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -184,15 +185,19 @@ public Optional findResourceRoleById(@NonNull Long id) { public List listByResourceTypeAndResourceId(ResourceType resourceType, Long resourceId) { List userResourceRoles = fromEntities(userResourceRoleRepository.listByResourceTypeAndId(resourceType, resourceId)); + if (resourceType == ResourceType.ODC_DATABASE) { + return userResourceRoles; + } List globalResourceRoles = globalResourceRoleService .findGlobalResourceRoleUsersByOrganizationId(authenticationFacade.currentOrganizationId()); if (CollectionUtils.isEmpty(globalResourceRoles)) { return userResourceRoles; } + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); globalResourceRoles.stream().map( i -> new UserResourceRole(i.getUserId(), resourceId, ResourceType.ODC_PROJECT, i.getResourceRole(), - true)) + resourceRoleName2Id.get(i.getResourceRole()), true)) .forEach(userResourceRoles::add); return userResourceRoles; } @@ -203,15 +208,19 @@ public List listByResourceTypeAndResourceIdIn(ResourceType res @NotEmpty Collection resourceIds) { List userResourceRoles = fromEntities(userResourceRoleRepository.listByResourceTypeAndIdIn(resourceType, resourceIds)); + if (resourceType == ResourceType.ODC_DATABASE) { + return userResourceRoles; + } List globalResourceRoles = globalResourceRoleService .findGlobalResourceRoleUsersByOrganizationId(authenticationFacade.currentOrganizationId()); if (CollectionUtils.isEmpty(globalResourceRoles)) { return userResourceRoles; } + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); globalResourceRoles.stream().flatMap(i -> resourceIds.stream().map( resourceId -> new UserResourceRole(i.getUserId(), resourceId, ResourceType.ODC_PROJECT, - i.getResourceRole(), true))) + i.getResourceRole(), resourceRoleName2Id.get(i.getResourceRole()), true))) .forEach(userResourceRoles::add); return userResourceRoles; } @@ -257,10 +266,11 @@ public List listByOrganizationIdAndUserId(Long organizationId, if (CollectionUtils.isEmpty(globalResourceRoles)) { return userResourceRoles; } - projectRepository.findAllByOrganizationId(organizationId).stream().filter(p -> !p.getArchived()) + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); + projectRepository.findAllByOrganizationId(organizationId).stream() .forEach(p -> globalResourceRoles.stream() .map(i -> new UserResourceRole(i.getUserId(), p.getId(), ResourceType.ODC_PROJECT, - i.getResourceRole(), true)) + i.getResourceRole(), resourceRoleName2Id.get(i.getResourceRole()), true)) .forEach(userResourceRoles::add)); return userResourceRoles; } @@ -275,10 +285,11 @@ public List listByUserId(Long userId) { if (CollectionUtils.isEmpty(globalResourceRoles)) { return userResourceRoles; } + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); projectRepository.findAllByOrganizationId(authenticationFacade.currentOrganizationId()).stream() - .filter(p -> !p.getArchived()).forEach(p -> globalResourceRoles.stream() + .forEach(p -> globalResourceRoles.stream() .map(i -> new UserResourceRole(i.getUserId(), p.getId(), ResourceType.ODC_PROJECT, - i.getResourceRole(), true)) + i.getResourceRole(), resourceRoleName2Id.get(i.getResourceRole()), true)) .forEach(userResourceRoles::add)); return userResourceRoles; } @@ -288,17 +299,67 @@ public List listByResourceIdAndTypeAndName(Long resourceId, Re String roleName) { List userResourceRoles = fromEntities( userResourceRoleRepository.findByResourceIdAndTypeAndName(resourceId, resourceType, roleName)); + if (resourceType == ResourceType.ODC_DATABASE) { + return userResourceRoles; + } List globalResourceRoles = globalResourceRoleService.findGlobalResourceRoleUsersByOrganizationIdAndRole( authenticationFacade.currentOrganizationId(), resourceType, ResourceRoleName.valueOf(roleName)); if (CollectionUtils.isEmpty(globalResourceRoles)) { return userResourceRoles; } + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); globalResourceRoles.stream().map(i -> new UserResourceRole(i.getUserId(), resourceId, resourceType, - i.getResourceRole(), true)).forEach(userResourceRoles::add); + i.getResourceRole(), resourceRoleName2Id.get(i.getResourceRole()), true)) + .forEach(userResourceRoles::add); return userResourceRoles; } + @SkipAuthorize("internal usage") + public List listByResourceIdentifierIn(Set resourceIdentifiers) { + List userResourceRoles = + fromEntities(userResourceRoleRepository.findByResourceIdsAndResourceRoleIdsIn(resourceIdentifiers)); + List globalUserResourceRoles = globalResourceRoleService + .findGlobalResourceRoleUsersByOrganizationIdAndRoleIn(authenticationFacade.currentOrganizationId(), + filterResourceRoleNames(ResourceType.ODC_PROJECT, resourceIdentifiers)); + if (CollectionUtils.isEmpty(globalUserResourceRoles)) { + return userResourceRoles; + } + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); + projectRepository.findAllByOrganizationId(authenticationFacade.currentOrganizationId()).stream() + .forEach(p -> globalUserResourceRoles.stream() + .map(i -> new UserResourceRole(i.getUserId(), p.getId(), ResourceType.ODC_PROJECT, + i.getResourceRole(), resourceRoleName2Id.get(i.getResourceRole()), true)) + .forEach(userResourceRoles::add)); + return userResourceRoles; + } + + private Map getProjectResourceRoleName2Id() { + return resourceRoleRepository.findByResourceType(ResourceType.ODC_PROJECT).stream().collect(Collectors.toMap( + ResourceRoleEntity::getRoleName, ResourceRoleEntity::getId, (v1, v2) -> v2)); + } + + private Set filterResourceRoleNames(ResourceType resourceType, Set resourceIdentifiers) { + if (CollectionUtils.isEmpty(resourceIdentifiers)) { + return Collections.emptySet(); + } + Map id2ResourceRoleName = resourceRoleRepository.findByResourceType(resourceType) + .stream() + .collect(Collectors.toMap(ResourceRoleEntity::getId, ResourceRoleEntity::getRoleName, (v1, v2) -> v2)); + Set filtered = new HashSet<>(); + resourceIdentifiers.stream().forEach(identifier -> { + String[] parts = identifier.split(":"); + if (parts.length != 2) { + throw new UnexpectedException("invalid resource identifier, identifier=" + identifier); + } + Long roleId = Long.parseLong(parts[1]); + if (id2ResourceRoleName.containsKey(roleId)) { + filtered.add(id2ResourceRoleName.get(roleId)); + } + }); + return filtered; + } + private List fromEntities(Collection entities) { if (CollectionUtils.isEmpty(entities)) { return new ArrayList<>(); @@ -320,7 +381,16 @@ private UserResourceRole fromEntity(UserResourceRoleEntity entity, ResourceRoleE model.setResourceType(resourceRole.getResourceType()); model.setResourceId(entity.getResourceId()); model.setUserId(entity.getUserId()); + model.setResourceRoleId(resourceRole.getId()); return model; } + public static UserResourceRoleEntity toEntity(UserResourceRole model) { + UserResourceRoleEntity entity = new UserResourceRoleEntity(); + entity.setResourceId(model.getResourceId()); + entity.setUserId(model.getUserId()); + entity.setResourceRoleId(model.getResourceRoleId()); + return entity; + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java index 0ed97ce37a..e27222b67b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java @@ -60,9 +60,9 @@ import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.metadb.iam.UserRepository; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.iam.ResourcePermissionExtractor; import com.oceanbase.odc.service.iam.ResourceRoleBasedPermissionExtractor; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.model.User; import com.oceanbase.odc.service.resourcegroup.model.ResourceIdentifier; @@ -79,7 +79,7 @@ public abstract class DefaultAuthorizationFacade implements AuthorizationFacade @Autowired private PermissionRepository repository; @Autowired - private UserResourceRoleRepository resourceRoleService; + private ResourceRoleService resourceRoleService; @Autowired private UserRepository userRepository; @Autowired @@ -205,9 +205,9 @@ private List getAllPermissions(Principal principal) { .stream().filter(permission -> !Objects.isNull(permission)).collect(Collectors.toList()); List resourceRoles = resourceRoleService - .findByOrganizationIdAndUserId(authenticationFacade.currentOrganizationId(), odcUser.getId()) + .listByOrganizationIdAndUserId(authenticationFacade.currentOrganizationId(), odcUser.getId()) .stream() - .filter(Objects::nonNull).collect(Collectors.toList()); + .filter(Objects::nonNull).map(ResourceRoleService::toEntity).collect(Collectors.toList()); return ListUtils.union(permissionMapper.getResourcePermissions(permissionEntityList), resourceRoleBasedPermissionExtractor.getResourcePermissions(resourceRoles)); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java index 25477cbbbe..2a3ae8b03f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java @@ -23,8 +23,8 @@ import com.oceanbase.odc.core.authority.permission.Permission; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.iam.ResourceRoleBasedPermissionExtractor; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.model.User; /** @@ -33,12 +33,12 @@ * @Description: [] */ public class ResourceRoleAuthorizer extends BaseAuthorizer { - protected final UserResourceRoleRepository repository; + protected final ResourceRoleService resourceRoleService; protected final ResourceRoleBasedPermissionExtractor permissionMapper; - public ResourceRoleAuthorizer(UserResourceRoleRepository repository, + public ResourceRoleAuthorizer(ResourceRoleService resourceRoleService, ResourceRoleBasedPermissionExtractor permissionMapper) { - this.repository = repository; + this.resourceRoleService = resourceRoleService; this.permissionMapper = permissionMapper; } @@ -52,8 +52,8 @@ protected List listPermittedPermissions(Principal principal) { * find all user-related resource role, and implies with permissions respectively */ List resourceRoles = - repository.findByUserId(odcUser.getId()).stream() - .filter(Objects::nonNull).collect(Collectors.toList()); + resourceRoleService.listByUserId(odcUser.getId()).stream() + .filter(Objects::nonNull).map(ResourceRoleService::toEntity).collect(Collectors.toList()); if (resourceRoles.isEmpty()) { return Collections.emptyList(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java index 83999e7539..d1ee19428c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java @@ -44,6 +44,8 @@ public class UserResourceRole implements PermissionConfiguration { private ResourceRoleName resourceRole; + private Long resourceRoleId; + private boolean derivedFromGlobalProjectRole = false; public boolean isProjectMember() { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/util/GlobalResourceRoleUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/util/GlobalResourceRoleUtil.java index 22f955aeca..77b82b9f52 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/util/GlobalResourceRoleUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/util/GlobalResourceRoleUtil.java @@ -17,6 +17,8 @@ import java.util.HashMap; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; @@ -52,4 +54,8 @@ public static String getGlobalRoleName(ResourceRoleName resourceRoleName) { return resourceRoleName2GlobalRoleName.get(resourceRoleName); } + public static Set getGlobalRoleName(Set resourceRoleNames) { + return resourceRoleNames.stream().map(resourceRoleName2GlobalRoleName::get).collect(Collectors.toSet()); + } + } From 620e98811437cf7ff31583ad7f50133b8f6ef1b7 Mon Sep 17 00:00:00 2001 From: zhangxiao <140503120+PeachThinking@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:48:17 +0800 Subject: [PATCH 084/118] fix(structure-compare):the structure synchronization task cannot be initiated when the structure comparison task is not created by yourself #4122 --- .../service/structurecompare/StructureComparisonService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java index 092505140b..436fc68d46 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java @@ -106,7 +106,7 @@ public DBStructureComparisonResp getDBStructureComparisonResult(@NonNull Long id try { StorageObject storageObject = objectStorageFacade.loadObject("structure-comparison".concat(File.separator) - .concat(authenticationFacade.currentUserIdStr()), taskEntity.getStorageObjectId()); + .concat(String.valueOf(taskEntity.getCreatorId())), taskEntity.getStorageObjectId()); Validate.notNull(storageObject, "StorageObject can not be null"); Validate.notNull(storageObject.getMetadata(), "ObjectMetadata can not be null"); if (storageObject.getMetadata().getTotalLength() > MAX_TOTAL_SCRIPT_SIZE_BYTES) { From 934a6055ec33184f6536b22237e25ca8faf54958 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Tue, 7 Jan 2025 14:28:33 +0800 Subject: [PATCH 085/118] fix(permission): global project roles cannot operate database/table permission apply and schedule tickets #4124 --- .../iam/resourcerole/UserResourceRoleRepository.java | 2 -- .../oceanbase/odc/service/iam/ResourceRoleService.java | 7 +++++++ .../permission/database/ApplyDatabaseFlowableTask.java | 9 ++++----- .../service/permission/table/ApplyTableFlowableTask.java | 9 ++++----- .../schedule/factory/ScheduleResponseMapperFactory.java | 2 ++ .../odc/service/schedule/model/ScheduleDetailResp.java | 3 +++ .../service/schedule/model/ScheduleDetailRespHist.java | 2 ++ 7 files changed, 22 insertions(+), 12 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java index f6d9b96c28..d860cb0edc 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java @@ -35,8 +35,6 @@ public interface UserResourceRoleRepository extends OdcJpaRepository findByUserId(Long userId); - List findByResourceId(Long resourceId); - @Modifying @Transactional @Query(value = "delete from iam_user_resource_role t where t.user_id =:userId", nativeQuery = true) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java index 178ada63d8..75a1c229e0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java @@ -202,6 +202,13 @@ public List listByResourceTypeAndResourceId(ResourceType resou return userResourceRoles; } + @Transactional(rollbackFor = Exception.class) + @SkipAuthorize("internal usage") + public Set listUserIdsByResourceTypeAndResourceId(ResourceType resourceType, Long resourceId) { + return listByResourceTypeAndResourceId(resourceType, resourceId).stream().map(UserResourceRole::getUserId) + .collect(Collectors.toSet()); + } + @Transactional(rollbackFor = Exception.class) @SkipAuthorize("internal usage") public List listByResourceTypeAndResourceIdIn(ResourceType resourceType, diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java index 02b20fc9aa..1fe24d7a95 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java @@ -44,11 +44,10 @@ import com.oceanbase.odc.metadb.iam.UserPermissionEntity; import com.oceanbase.odc.metadb.iam.UserPermissionRepository; import com.oceanbase.odc.metadb.iam.UserRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.connection.database.model.DatabaseType; import com.oceanbase.odc.service.flow.task.BaseODCFlowTaskDelegate; import com.oceanbase.odc.service.flow.util.FlowTaskUtil; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseParameter; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseParameter.ApplyDatabase; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseResult; @@ -77,7 +76,7 @@ public class ApplyDatabaseFlowableTask extends BaseODCFlowTaskDelegate projectMemberIds = userResourceRoleRepository.findByResourceId(projectId).stream() - .map(UserResourceRoleEntity::getUserId).collect(Collectors.toSet()); + Set projectMemberIds = + resourceRoleService.listUserIdsByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, projectId); if (!projectMemberIds.contains(this.creatorId)) { log.warn("User not member of project, userId={}, projectId={}", this.creatorId, projectId); throw new IllegalStateException("User not member of project"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java index 114c2693c7..b520ef83c1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java @@ -45,10 +45,9 @@ import com.oceanbase.odc.metadb.iam.UserPermissionEntity; import com.oceanbase.odc.metadb.iam.UserPermissionRepository; import com.oceanbase.odc.metadb.iam.UserRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.flow.task.BaseODCFlowTaskDelegate; import com.oceanbase.odc.service.flow.util.FlowTaskUtil; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter; import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter.ApplyTable; @@ -87,7 +86,7 @@ public class ApplyTableFlowableTask extends BaseODCFlowTaskDelegate projectMemberIds = userResourceRoleRepository.findByResourceId(projectId).stream() - .map(UserResourceRoleEntity::getUserId).collect(Collectors.toSet()); + Set projectMemberIds = + resourceRoleService.listUserIdsByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, projectId); if (!projectMemberIds.contains(this.creatorId)) { log.warn("User not member of project, userId={}, projectId={}", this.creatorId, projectId); throw new IllegalStateException("User not member of project"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java index 06db052a84..d61c3a42cf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java @@ -134,6 +134,7 @@ public ScheduleDetailResp generateScheduleDetailResp(@NonNull Schedule schedule) scheduleDetailResp.setCreateTime(schedule.getCreateTime()); scheduleDetailResp.setUpdateTime(schedule.getUpdateTime()); scheduleDetailResp.setProjectId(schedule.getProjectId()); + scheduleDetailResp.setProject(projectService.detail(schedule.getProjectId())); scheduleDetailResp.setDescription(schedule.getDescription()); scheduleDetailResp.setNextFireTimes( @@ -262,6 +263,7 @@ public ScheduleDetailRespHist generateHistoryScheduleDetail(Schedule schedule) { resp.setStatus(schedule.getStatus()); resp.setProjectId(schedule.getProjectId()); + resp.setProject(projectService.detail(schedule.getProjectId())); resp.setJobParameters(schedule.getParameters()); resp.setTriggerConfig(schedule.getTriggerConfig()); resp.setNextFireTimes(QuartzCronExpressionUtils.getNextFiveFireTimes(schedule.getTriggerConfig())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailResp.java index 4f2fdb67ee..ee557bec21 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailResp.java @@ -19,6 +19,7 @@ import java.util.List; import com.oceanbase.odc.common.i18n.Internationalizable; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.model.InnerUser; import com.oceanbase.odc.service.quartz.model.MisfireStrategy; @@ -62,4 +63,6 @@ public class ScheduleDetailResp { private TriggerConfig triggerConfig; + private Project project; + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailRespHist.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailRespHist.java index 35106ff4e4..b968ca9e63 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailRespHist.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailRespHist.java @@ -23,6 +23,7 @@ import com.oceanbase.odc.core.shared.OrganizationIsolated; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.metadb.flow.FlowInstanceEntity; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.model.InnerUser; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.quartz.model.MisfireStrategy; @@ -73,6 +74,7 @@ public class ScheduleDetailRespHist implements OrganizationIsolated { private Set candidateApprovers; private List jobs; private ScheduleTaskParameters jobParameters; + private Project project; @Override public String resourceType() { From c0de17785edafb54b5f391e1a50316ef27f1d25e Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:31:14 +0800 Subject: [PATCH 086/118] fix(session): kill session may lead npe #4114 --- .../service/db/session/DefaultDBSessionManage.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index 99c0bb22fa..e609a56150 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -39,6 +39,8 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import javax.validation.constraints.NotNull; + import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.SetUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -232,8 +234,8 @@ private List executeSqls(ConnectionSession connectionSession, private List additionalKillIfNecessary(ConnectionSession connectionSession, List results, List sqlTupleSessionIds) { Map sessionId2SvrAddr = - getSessionList(connectionSession, null).stream().collect( - Collectors.toMap(OdcDBSession::getSessionId, + getSessionList(connectionSession, s -> s.getSvrIp() != null) + .stream().collect(Collectors.toMap(OdcDBSession::getSessionId, s -> extractServerAddress(MoreObjects.firstNonNull(s.getSvrIp(), "")))); Map sqlId2SessionId = sqlTupleSessionIds.stream().collect( Collectors.toMap(s -> s.getSqlTuple().getSqlId(), SqlTupleSessionId::getSessionId)); @@ -423,22 +425,23 @@ private void directLinkServerAndExecute(String sql, ConnectionSession session, S // extract text(query from the dictionary) to server address(ip, port) // the text is expected be like 0.0.0.0:8888 + @NotNull private ServerAddress extractServerAddress(String text) { String trimmed = StringUtils.trim(text); if (StringUtils.isBlank(trimmed)) { log.info("unable to extract server address, text is empty"); - return null; + throw new IllegalStateException("Empty server address!"); } Matcher matcher = SERVER_PATTERN.matcher(trimmed); if (!matcher.matches()) { log.info("unable to extract server address, does not match pattern"); - return null; + throw new IllegalStateException("Invalid server address!"); } String ipAddress = matcher.group("ip"); String port = matcher.group("port"); if (StringUtils.isEmpty(ipAddress) || StringUtils.isEmpty(port)) { log.info("unable to extract server address, ipAddress={}, port={}", ipAddress, port); - return null; + throw new IllegalStateException("Invalid server address!"); } return new ServerAddress(ipAddress, port); } From 1a216cdb5da4bd177e996ee4228c1ac90f518912 Mon Sep 17 00:00:00 2001 From: LioRoger Date: Tue, 7 Jan 2025 15:51:54 +0800 Subject: [PATCH 087/118] fix(task): log4j set failed for task --- server/odc-server/src/main/resources/log4j2-task.xml | 10 ++++++++++ .../odc/service/task/caller/ProcessJobCaller.java | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/server/odc-server/src/main/resources/log4j2-task.xml b/server/odc-server/src/main/resources/log4j2-task.xml index b79d3d78bf..d99b722e56 100644 --- a/server/odc-server/src/main/resources/log4j2-task.xml +++ b/server/odc-server/src/main/resources/log4j2-task.xml @@ -55,6 +55,11 @@ [%d{yyyy-MM-dd HH:mm:ss z}] [%X{taskId}] [%p] %m%n + + + + + @@ -73,6 +78,11 @@ [%d{yyyy-MM-dd HH:mm:ss z}] [%X{taskId}] [%p] %m%n + + + + + diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java index 090ff3abf4..c976c672c6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java @@ -18,12 +18,15 @@ import static com.oceanbase.odc.service.task.constants.JobConstants.ODC_EXECUTOR_CANNOT_BE_DESTROYED; +import java.io.File; import java.io.IOException; +import java.lang.ProcessBuilder.Redirect; import java.text.MessageFormat; import java.util.Objects; import java.util.Optional; import com.fasterxml.jackson.core.type.TypeReference; +import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.service.common.response.OdcResult; @@ -58,6 +61,10 @@ public ExecutorIdentifier doStart(JobContext context) throws JobException { String executorName = JobUtils.generateExecutorName(context.getJobIdentity()); ProcessBuilder pb = new ExecutorProcessBuilderFactory().getProcessBuilder( processConfig, context.getJobIdentity().getId(), executorName); + log.info("start task with processConfig={}, env={}", JobUtils.toJson(processConfig), + JsonUtils.toJson(pb.environment())); + pb.redirectErrorStream(true); + pb.redirectOutput(Redirect.appendTo(new File("process-call.log"))); Process process; try { process = pb.start(); From 45b9f155aeebe324dc3198156b6b60285e658790 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Tue, 7 Jan 2025 15:57:14 +0800 Subject: [PATCH 088/118] build: update 4.3.3 submodule #4126 --- client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client b/client index 0021be9fee..97074b7ddf 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 0021be9feed6d121c190f1aa9565c0ab9fa34dc5 +Subproject commit 97074b7ddffb212b0a43687bb88665d94450eb7b From 43b45fa27b5316fc3c2cc6aaedf145384d886616 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Wed, 8 Jan 2025 12:01:42 +0800 Subject: [PATCH 089/118] fix(permission): external approval integration doesn't take effect #4128 --- .../odc/service/flow/FlowInstanceService.java | 29 +++++++++++-------- .../odc/service/iam/ResourceRoleService.java | 26 ++++++++--------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java index 76620d1853..3fa3f737da 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java @@ -1072,18 +1072,9 @@ private FlowInstanceConfigurer buildConfigurer( private void completeApprovalInstance(@NonNull Long flowInstanceId, @NonNull Consumer consumer, Boolean skipAuth) { List instances = - mapFlowInstanceWithApprovalPermission(flowInstanceId, - flowInstance -> flowInstance.filterInstanceNode(instance -> { - if (instance.getNodeType() != FlowNodeType.APPROVAL_TASK) { - return false; - } - return instance.getStatus() == FlowNodeStatus.EXECUTING - || instance.getStatus() == FlowNodeStatus.WAIT_FOR_CONFIRM; - }).stream().map(instance -> { - Verify.verify(instance instanceof FlowApprovalInstance, - "FlowApprovalInstance's type is illegal"); - return (FlowApprovalInstance) instance; - }).collect(Collectors.toList())); + skipAuth ? mapFlowInstanceWithoutPermissionCheck(flowInstanceId, + generateApprovalMapper()) + : mapFlowInstanceWithApprovalPermission(flowInstanceId, generateApprovalMapper()); PreConditions.validExists(ResourceType.ODC_FLOW_APPROVAL_INSTANCE, "flowInstanceId", flowInstanceId, () -> instances.size() > 0); Verify.singleton(instances, "ApprovalInstance"); @@ -1092,6 +1083,20 @@ private void completeApprovalInstance(@NonNull Long flowInstanceId, consumer.accept(target); } + private Function> generateApprovalMapper() { + return flowInstance -> flowInstance.filterInstanceNode(instance -> { + if (instance.getNodeType() != FlowNodeType.APPROVAL_TASK) { + return false; + } + return instance.getStatus() == FlowNodeStatus.EXECUTING + || instance.getStatus() == FlowNodeStatus.WAIT_FOR_CONFIRM; + }).stream().map(instance -> { + Verify.verify(instance instanceof FlowApprovalInstance, + "FlowApprovalInstance's type is illegal"); + return (FlowApprovalInstance) instance; + }).collect(Collectors.toList()); + } + private void initVariables(Map variables, TaskEntity taskEntity, TaskEntity preCheckTaskEntity, List configList, RiskLevelDescriber riskLevelDescriber) { FlowTaskUtil.setTaskId(variables, taskEntity.getId()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java index 75a1c229e0..5faca9a970 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java @@ -134,12 +134,11 @@ public Set getResourceRoleIdentifiersByUserId(long organizationId, long Map resourceRoleName2Id = resourceRoleRepository.findByResourceType(ResourceType.ODC_PROJECT) .stream().map(resourceRoleMapper::entityToModel) .collect(Collectors.toMap(role -> role.getRoleName().name(), ResourceRole::getId, (v1, v2) -> v2)); - - Set derivedFromGlobalResourceRole = globalResourceRoles.stream() - .map(i -> StringUtils.join("*", ":", resourceRoleName2Id.get(i.name()))) - .collect(Collectors.toSet()); - derivedFromGlobalResourceRole.addAll(resourceRoleIdentifiers); - return derivedFromGlobalResourceRole; + projectRepository.findAllByOrganizationId(organizationId).stream() + .forEach(p -> globalResourceRoles.stream() + .map(r -> StringUtils.join(p.getId(), ":", resourceRoleName2Id.get(r.name()))) + .forEach(resourceRoleIdentifiers::add)); + return resourceRoleIdentifiers; } @SkipAuthorize @@ -164,14 +163,13 @@ public Map> getProjectId2ResourceRoleNames(Long user if (CollectionUtils.isEmpty(globalResourceRoles)) { return result; } - projectRepository.findAllByOrganizationId(organizationId).stream().filter(p -> !p.getArchived()) - .forEach(p -> { - if (!result.containsKey(p.getId())) { - result.put(p.getId(), globalResourceRoles); - } else { - result.get(p.getId()).addAll(globalResourceRoles); - } - }); + projectRepository.findAllByOrganizationId(organizationId).stream().forEach(p -> { + if (!result.containsKey(p.getId())) { + result.put(p.getId(), globalResourceRoles); + } else { + result.get(p.getId()).addAll(globalResourceRoles); + } + }); return result; } From ad337c37507fd35bfc992d54ec6989edcdfa7a56 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:59:09 +0800 Subject: [PATCH 090/118] fix(session): oracle mode effected by kill-query-or-session.max-supported-ob-version #4134 * oracle mode effected by kill-query-or-session.max-supported-ob-version * oracle mode effected by kill-query-or-session.max-supported-ob-version --- .../odc/service/feature/VersionDiffConfigService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java index ee5d4f2cf6..c3947af58a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java @@ -136,8 +136,10 @@ public List getSupportFeatures(ConnectionSession connectionSession) { } // killSession that is greater than the specified version is currently not supported - if (SUPPORT_KILL_SESSION.equalsIgnoreCase(configKey) - || SUPPORT_KILL_QUERY.equalsIgnoreCase(configKey)) { + // Only effect OB mode dialect + if (connectionSession.getDialectType().isOceanbase() && + (SUPPORT_KILL_SESSION.equalsIgnoreCase(configKey) + || SUPPORT_KILL_QUERY.equalsIgnoreCase(configKey))) { Optional nonSupport = systemConfigs.stream().filter( c -> c.getKey().equalsIgnoreCase(MAX_SUPPORT_KILL_OB_VERSION)).findFirst(); if (nonSupport.isPresent()) { From 7adb6261b6fa9fd67ae5f22a8f8921d0fecdffc6 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Thu, 9 Jan 2025 14:08:42 +0800 Subject: [PATCH 091/118] fix(audit): update i18n resources and add archive/delete projects audit events #4141 * fix audit * fix i18n * response to comments --- .../core/shared/constant/AuditEventAction.java | 4 ++++ .../resources/i18n/BusinessMessages.properties | 12 +++++++----- .../i18n/BusinessMessages_zh_CN.properties | 16 +++++++++------- .../i18n/BusinessMessages_zh_TW.properties | 16 +++++++++------- .../init-config/default-audit-event-meta.yml | 16 ++++++++++++++++ 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java index 32bcc74d2d..9ceccde9f4 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java @@ -349,6 +349,10 @@ public enum AuditEventAction implements Translatable { CREATE_PROJECT, + ARCHIVE_PROJECT, + + DELETE_PROJECT, + CREATE_ENVIRONMENT, UPDATE_ENVIRONMENT, diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties index ccc8a223ae..e7d335972b 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties @@ -224,7 +224,7 @@ com.oceanbase.odc.AuditEventAction.CREATE_ALTER_SCHEDULE_TASK=Create alter sched com.oceanbase.odc.AuditEventAction.CREATE_ONLINE_SCHEMA_CHANGE_TASK=Create online schema change task com.oceanbase.odc.AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK=Create apply project permission task com.oceanbase.odc.AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK=Create apply database permission task -com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=Create apply table permission task +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=Create table/view permission application com.oceanbase.odc.AuditEventAction.STOP_ASYNC_TASK=Stop database change task com.oceanbase.odc.AuditEventAction.STOP_MULTIPLE_ASYNC_TASK=Stop batch database change task com.oceanbase.odc.AuditEventAction.STOP_MOCKDATA_TASK=Stop mock data task @@ -238,7 +238,7 @@ com.oceanbase.odc.AuditEventAction.STOP_ALTER_SCHEDULE_TASK=Stop alter schedule com.oceanbase.odc.AuditEventAction.STOP_ONLINE_SCHEMA_CHANGE_TASK=Stop online schema change task com.oceanbase.odc.AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK=Stop apply project permission task com.oceanbase.odc.AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK=Stop apply database permission task -com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=Stop apply table permission task +com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=Cancel table/view permission application com.oceanbase.odc.AuditEventAction.EXECUTE_ASYNC_TASK=Execute database change task com.oceanbase.odc.AuditEventAction.EXECUTE_MULTIPLE_ASYNC_TASK=Execute batch database change task com.oceanbase.odc.AuditEventAction.EXECUTE_MOCKDATA_TASK=Execute mock data task @@ -264,7 +264,7 @@ com.oceanbase.odc.AuditEventAction.APPROVE_ALTER_SCHEDULE_TASK=Approve alter sch com.oceanbase.odc.AuditEventAction.APPROVE_ONLINE_SCHEMA_CHANGE_TASK=Approve online schema change task com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK=Approve apply project permission task com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK=Approve apply database permission task -com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=Approve apply table permission task +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=Approve table/view permission application com.oceanbase.odc.AuditEventAction.REJECT_ASYNC_TASK=Reject database change task com.oceanbase.odc.AuditEventAction.REJECT_MULTIPLE_ASYNC_TASK=Reject batch database change task com.oceanbase.odc.AuditEventAction.REJECT_MOCKDATA_TASK=Reject mock data task @@ -279,7 +279,7 @@ com.oceanbase.odc.AuditEventAction.REJECT_ALTER_SCHEDULE_TASK=Reject alter sched com.oceanbase.odc.AuditEventAction.REJECT_ONLINE_SCHEMA_CHANGE_TASK=Reject online schema change task com.oceanbase.odc.AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK=Reject apply project permission task com.oceanbase.odc.AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK=Reject apply database permission task -com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=Reject apply table permission task +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=Reject table/view permission application com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_RULE=Create data masking rule com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_RULE=Update data masking rule com.oceanbase.odc.AuditEventAction.ENABLE_DATA_MASKING_RULE=Enable data masking rule @@ -298,6 +298,8 @@ com.oceanbase.odc.AuditEventAction.CREATE_DATASOURCE=Create datasource com.oceanbase.odc.AuditEventAction.DELETE_DATASOURCE=delete datasource com.oceanbase.odc.AuditEventAction.UPDATE_DATASOURCE=Update datasource com.oceanbase.odc.AuditEventAction.CREATE_PROJECT=Create project +com.oceanbase.odc.AuditEventAction.ARCHIVE_PROJECT=Archive project +com.oceanbase.odc.AuditEventAction.DELETE_PROJECT=Delete project com.oceanbase.odc.AuditEventAction.UPDATE_SQL_SECURITY_RULE=Modify SQL security rule com.oceanbase.odc.AuditEventAction.GRANT_DATABASE_PERMISSION=Grant database permission com.oceanbase.odc.AuditEventAction.REVOKE_DATABASE_PERMISSION=Revoke database permission @@ -352,7 +354,7 @@ com.oceanbase.odc.AuditEventType.ALTER_SCHEDULE=Alter schedule com.oceanbase.odc.AuditEventType.ONLINE_SCHEMA_CHANGE=Online schema change com.oceanbase.odc.AuditEventType.APPLY_PROJECT_PERMISSION=Apply project permission com.oceanbase.odc.AuditEventType.APPLY_DATABASE_PERMISSION=Apply database permission -com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=Apply table permission +com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=Apply for table/view permissions com.oceanbase.odc.AuditEventType.DATA_MASKING_RULE=Data masking rule com.oceanbase.odc.AuditEventType.DATA_MASKING_POLICY=Data masking policy com.oceanbase.odc.AuditEventType.PERMISSION_APPLY=Permission apply diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties index 0dc888df39..280ff945ba 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties @@ -223,7 +223,7 @@ com.oceanbase.odc.AuditEventAction.CREATE_ALTER_SCHEDULE_TASK=创建修改调度 com.oceanbase.odc.AuditEventAction.CREATE_ONLINE_SCHEMA_CHANGE_TASK=创建无锁结构变更任务 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK=创建申请项目权限任务 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK=创建申请数据库权限任务 -com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=创建申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=创建申请数据表/视图权限任务 com.oceanbase.odc.AuditEventAction.STOP_ASYNC_TASK=停止数据库变更任务 com.oceanbase.odc.AuditEventAction.STOP_MULTIPLE_ASYNC_TASK=停止多库变更任务 com.oceanbase.odc.AuditEventAction.STOP_MOCKDATA_TASK=停止模拟数据任务 @@ -237,7 +237,7 @@ com.oceanbase.odc.AuditEventAction.STOP_ALTER_SCHEDULE_TASK=停止修改调度 com.oceanbase.odc.AuditEventAction.STOP_ONLINE_SCHEMA_CHANGE_TASK=停止无锁结构变更任务 com.oceanbase.odc.AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK=停止申请项目权限任务 com.oceanbase.odc.AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK=停止申请数据库权限任务 -com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=停止申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=停止申请数据表/视图权限任务 com.oceanbase.odc.AuditEventAction.EXECUTE_ASYNC_TASK=执行数据库变更任务 com.oceanbase.odc.AuditEventAction.EXECUTE_MULTIPLE_ASYNC_TASK=执行多库变更任务 com.oceanbase.odc.AuditEventAction.EXECUTE_MOCKDATA_TASK=执行模拟数据任务 @@ -263,7 +263,7 @@ com.oceanbase.odc.AuditEventAction.APPROVE_ALTER_SCHEDULE_TASK=同意修改调 com.oceanbase.odc.AuditEventAction.APPROVE_ONLINE_SCHEMA_CHANGE_TASK=同意无锁结构变更任务 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK=同意申请项目权限任务 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK=同意申请数据库权限任务 -com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=同意申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=同意申请数据表/视图权限任务 com.oceanbase.odc.AuditEventAction.REJECT_ASYNC_TASK=拒绝数据库变更任务 com.oceanbase.odc.AuditEventAction.REJECT_MULTIPLE_ASYNC_TASK=拒绝多库变更任务 com.oceanbase.odc.AuditEventAction.REJECT_MOCKDATA_TASK=拒绝模拟数据任务 @@ -278,7 +278,7 @@ com.oceanbase.odc.AuditEventAction.REJECT_ALTER_SCHEDULE_TASK=拒绝修改调度 com.oceanbase.odc.AuditEventAction.REJECT_ONLINE_SCHEMA_CHANGE_TASK=拒绝无锁结构变更任务 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK=拒绝申请项目权限任务 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK=拒绝申请数据库权限任务 -com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=拒绝申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=拒绝申请数据表/视图权限任务 com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_RULE=创建脱敏规则 com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_RULE=更新脱敏规则 com.oceanbase.odc.AuditEventAction.ENABLE_DATA_MASKING_RULE=启用脱敏规则 @@ -297,6 +297,8 @@ com.oceanbase.odc.AuditEventAction.CREATE_DATASOURCE=创建数据源 com.oceanbase.odc.AuditEventAction.DELETE_DATASOURCE=删除数据源 com.oceanbase.odc.AuditEventAction.UPDATE_DATASOURCE=更新数据源 com.oceanbase.odc.AuditEventAction.CREATE_PROJECT=创建项目 +com.oceanbase.odc.AuditEventAction.ARCHIVE_PROJECT=归档项目 +com.oceanbase.odc.AuditEventAction.DELETE_PROJECT=删除项目 com.oceanbase.odc.AuditEventAction.UPDATE_SQL_SECURITY_RULE=修改 SQL 安全规则 com.oceanbase.odc.AuditEventAction.GRANT_DATABASE_PERMISSION=新增库权限 com.oceanbase.odc.AuditEventAction.REVOKE_DATABASE_PERMISSION=回收库权限 @@ -357,7 +359,7 @@ com.oceanbase.odc.AuditEventType.ALTER_SCHEDULE=修改调度 com.oceanbase.odc.AuditEventType.ONLINE_SCHEMA_CHANGE=无锁结构变更 com.oceanbase.odc.AuditEventType.APPLY_PROJECT_PERMISSION=申请项目权限 com.oceanbase.odc.AuditEventType.APPLY_DATABASE_PERMISSION=申请数据库权限 -com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=申请数据表权限 +com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=申请数据表/视图权限 com.oceanbase.odc.AuditEventType.DATA_MASKING_RULE=脱敏规则 com.oceanbase.odc.AuditEventType.DATA_MASKING_POLICY=脱敏策略 com.oceanbase.odc.AuditEventType.PERMISSION_APPLY=权限申请 @@ -791,7 +793,7 @@ com.oceanbase.odc.TaskType.PRE_CHECK=预检查 com.oceanbase.odc.TaskType.EXPORT_RESULT_SET=导出结果集 com.oceanbase.odc.TaskType.APPLY_PROJECT_PERMISSION=申请项目权限 com.oceanbase.odc.TaskType.APPLY_DATABASE_PERMISSION=申请数据库权限 -com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=申请数据表权限 +com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=申请数据表/视图权限 com.oceanbase.odc.TaskType.SQL_PLAN=SQL 计划 com.oceanbase.odc.TaskType.DATA_ARCHIVE=数据归档 com.oceanbase.odc.TaskType.DATA_DELETE=数据清理 @@ -822,7 +824,7 @@ com.oceanbase.odc.notification.channel-test-message=【ODC】消息通道验证 # com.oceanbase.odc.builtin-resource.permission-apply.project.description=申请项目【{0}】的【{1}】权限 com.oceanbase.odc.builtin-resource.permission-apply.database.description=申请数据库的【{0}】权限 -com.oceanbase.odc.builtin-resource.permission-apply.table.description=申请数据表的【{0}】权限 +com.oceanbase.odc.builtin-resource.permission-apply.table.description=申请数据表/视图的【{0}】权限 # # Multiple Async diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties index 6d3b3ae0a9..93555afddc 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties @@ -224,7 +224,7 @@ com.oceanbase.odc.AuditEventAction.CREATE_ALTER_SCHEDULE_TASK=創建修改調度 com.oceanbase.odc.AuditEventAction.CREATE_ONLINE_SCHEMA_CHANGE_TASK=創建無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK=創建申請項目權限任務 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK=創建申請數據庫權限任務 -com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=創建申請數據表權限任務 +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=創建申請數據表/視圖權限任務 com.oceanbase.odc.AuditEventAction.STOP_ASYNC_TASK=停止數據庫變更任務 com.oceanbase.odc.AuditEventAction.STOP_MULTIPLE_ASYNC_TASK=停止多庫變更任務 com.oceanbase.odc.AuditEventAction.STOP_MOCKDATA_TASK=停止模擬數據任務 @@ -238,7 +238,7 @@ com.oceanbase.odc.AuditEventAction.STOP_ALTER_SCHEDULE_TASK=停止修改調度 com.oceanbase.odc.AuditEventAction.STOP_ONLINE_SCHEMA_CHANGE_TASK=停止無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK=停止申請項目權限任務 com.oceanbase.odc.AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK=停止申請數據庫權限任務 -com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=停止申請數據表權限任務 +com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=停止申請數據表/視圖權限任務 com.oceanbase.odc.AuditEventAction.EXECUTE_ASYNC_TASK=執行數據庫變更任務 com.oceanbase.odc.AuditEventAction.EXECUTE_MULTIPLE_ASYNC_TASK=執行多庫變更任務 com.oceanbase.odc.AuditEventAction.EXECUTE_MOCKDATA_TASK=執行模擬數據任務 @@ -264,7 +264,7 @@ com.oceanbase.odc.AuditEventAction.APPROVE_ALTER_SCHEDULE_TASK=同意修改調 com.oceanbase.odc.AuditEventAction.APPROVE_ONLINE_SCHEMA_CHANGE_TASK=同意無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK=同意申請項目權限任務 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK=同意申請數據庫權限任務 -com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=同意申請數據表權限任務 +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=同意申請數據表/視圖權限任務 com.oceanbase.odc.AuditEventAction.REJECT_ASYNC_TASK=拒絕數據庫變更任務 com.oceanbase.odc.AuditEventAction.REJECT_MULTIPLE_ASYNC_TASK=拒絕多庫變更任務 com.oceanbase.odc.AuditEventAction.REJECT_MOCKDATA_TASK=拒絕模擬數據任務 @@ -279,7 +279,7 @@ com.oceanbase.odc.AuditEventAction.REJECT_ALTER_SCHEDULE_TASK=拒絕修改調度 com.oceanbase.odc.AuditEventAction.REJECT_ONLINE_SCHEMA_CHANGE_TASK=拒絕無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK=拒絕申請項目權限任務 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK=拒絕申請數據庫權限任務 -com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=拒絕申請數據表權限任務 +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=拒絕申請數據表/視圖權限任務 com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_RULE=創建脫敏規則 com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_RULE=更新脫敏規則 com.oceanbase.odc.AuditEventAction.ENABLE_DATA_MASKING_RULE=啓用脫敏規則 @@ -298,6 +298,8 @@ com.oceanbase.odc.AuditEventAction.CREATE_DATASOURCE=創建數據庫源 com.oceanbase.odc.AuditEventAction.DELETE_DATASOURCE=删除數據庫源 com.oceanbase.odc.AuditEventAction.UPDATE_DATASOURCE=更新數據庫源 com.oceanbase.odc.AuditEventAction.CREATE_PROJECT=創建项目 +com.oceanbase.odc.AuditEventAction.ARCHIVE_PROJECT=歸檔項目 +com.oceanbase.odc.AuditEventAction.DELETE_PROJECT=刪除項目 com.oceanbase.odc.AuditEventAction.UPDATE_SQL_SECURITY_RULE=修改 SQL 安全規則 com.oceanbase.odc.AuditEventAction.GRANT_DATABASE_PERMISSION=新增庫權限 com.oceanbase.odc.AuditEventAction.REVOKE_DATABASE_PERMISSION=回收庫權限 @@ -357,7 +359,7 @@ com.oceanbase.odc.AuditEventType.STRUCTURE_COMPARISON=結構比對 com.oceanbase.odc.AuditEventType.ONLINE_SCHEMA_CHANGE=無鎖結構變更 com.oceanbase.odc.AuditEventType.APPLY_PROJECT_PERMISSION=申請項目權限 com.oceanbase.odc.AuditEventType.APPLY_DATABASE_PERMISSION=申請數據庫權限 -com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=申請數據表權限 +com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=申請數據表/視圖權限 com.oceanbase.odc.AuditEventType.DATA_MASKING_RULE=脫敏規則 com.oceanbase.odc.AuditEventType.DATA_MASKING_POLICY=脫敏策略 com.oceanbase.odc.AuditEventType.PERMISSION_APPLY=權限申請 @@ -861,7 +863,7 @@ com.oceanbase.odc.TaskType.PRE_CHECK=預檢查 com.oceanbase.odc.TaskType.EXPORT_RESULT_SET=導出結果集 com.oceanbase.odc.TaskType.APPLY_PROJECT_PERMISSION=申請項目權限 com.oceanbase.odc.TaskType.APPLY_DATABASE_PERMISSION=申請數據庫權限 -com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=申請數據表權限 +com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=申請數據表/視圖權限 com.oceanbase.odc.TaskType.SQL_PLAN=SQL 計劃 com.oceanbase.odc.TaskType.DATA_ARCHIVE=數據歸檔 com.oceanbase.odc.TaskType.DATA_DELETE=數據清理 @@ -892,7 +894,7 @@ com.oceanbase.odc.notification.channel-test-message=【ODC】消息通道驗證 # com.oceanbase.odc.builtin-resource.permission-apply.project.description=申請項目【{0}】的【{1}】權限 com.oceanbase.odc.builtin-resource.permission-apply.database.description=申請數據庫的【{0}】權限 -com.oceanbase.odc.builtin-resource.permission-apply.table.description=申請數據表的【{0}】權限 +com.oceanbase.odc.builtin-resource.permission-apply.table.description=申請數據表/視圖的【{0}】權限 # # Multiple Async diff --git a/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml b/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml index a5e85ac1de..dac3e86d48 100644 --- a/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml +++ b/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml @@ -555,3 +555,19 @@ in_connection: 0 enabled: 1 id: 100 + +- type: "PROJECT_MANAGEMENT" + action: "ARCHIVE_PROJECT" + method_signature: "com.oceanbase.odc.server.web.controller.v2.ProjectController.setArchived" + sid_extract_expression: "" + in_connection: 0 + enabled: 1 + id: 101 + +- type: "PROJECT_MANAGEMENT" + action: "DELETE_PROJECT" + method_signature: "com.oceanbase.odc.server.web.controller.v2.ProjectController.batchDelete" + sid_extract_expression: "" + in_connection: 0 + enabled: 1 + id: 102 From db311d598b6bb41979bd5557a14d91ce1faf519c Mon Sep 17 00:00:00 2001 From: LioRoger Date: Thu, 9 Jan 2025 14:21:51 +0800 Subject: [PATCH 092/118] feat(resource): add double check when destroy resource --- .../com/oceanbase/odc/service/resource/ResourceManager.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java index 60e9a3a7d4..3f5f3c59cd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java @@ -233,8 +233,11 @@ public void release(@NonNull ResourceID resourceID) { @SkipAuthorize("odc internal usage") public String destroy(@NonNull ResourceID resourceID) throws Exception { Optional optional = this.resourceRepository.findByResourceID(resourceID); - if (!optional.isPresent()) { + if (!optional.isPresent()) { // may old version job log.warn("Resource is not found, resourceID={}", resourceID); + } else if (optional.get().getStatus() == ResourceState.DESTROYING) { + log.warn("Resource is already in destroying state, resourceID={}", resourceID); + return null; } return doDestroy(resourceID); } From 25aa69459d9b55f5e0e9c64f2a317e3508378c89 Mon Sep 17 00:00:00 2001 From: kiko Date: Thu, 9 Jan 2025 15:36:36 +0800 Subject: [PATCH 093/118] fix(schedule):check has running task when delete (#4143) * fix:check has running task when delete * resp comments --- .../com/oceanbase/odc/service/schedule/ScheduleService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index e3a8fb33b5..6d8f302070 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -327,7 +327,8 @@ public ChangeScheduleResp changeSchedule(ScheduleChangeParams req) { } if (req.getOperationType() == OperationType.DELETE) { PreConditions.validRequestState(targetSchedule.getStatus() == ScheduleStatus.TERMINATED - || targetSchedule.getStatus() == ScheduleStatus.COMPLETED, ErrorCodes.DeleteNotAllowed, null, + || targetSchedule.getStatus() == ScheduleStatus.COMPLETED + || !hasExecutingTask(targetSchedule.getId()), ErrorCodes.DeleteNotAllowed, null, "Delete schedule is not allowed."); } } From 7411a76ef75e3a73d8d756bda9b096de96f7c1a5 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:08:54 +0800 Subject: [PATCH 094/118] client mode exclude saml (#4142) --- .../src/main/resources/config/application-clientMode.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/server/odc-server/src/main/resources/config/application-clientMode.yml b/server/odc-server/src/main/resources/config/application-clientMode.yml index 6ee099ef2a..395a1da8c9 100644 --- a/server/odc-server/src/main/resources/config/application-clientMode.yml +++ b/server/odc-server/src/main/resources/config/application-clientMode.yml @@ -21,6 +21,7 @@ spring: - org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration - org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration - org.springframework.boot.autoconfigure.security.oauth2.client.servlet.OAuth2ClientAutoConfiguration + - org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyAutoConfiguration - org.springframework.boot.actuate.autoconfigure.metrics.export.prometheus.PrometheusMetricsExportAutoConfiguration jpa: database-platform: org.hibernate.dialect.H2Dialect From 085058b09b20f65e055f090c2271c187a608b9eb Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Fri, 10 Jan 2025 09:58:01 +0800 Subject: [PATCH 095/118] fix(logicaldatabase): it may NPE when the logical database task starts --- .../LogicalDatabaseChangeTask.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java index eaa6723245..e99cdac2e9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -78,16 +79,16 @@ public class LogicalDatabaseChangeTask extends TaskBase jobParameters = context.getJobParameters(); taskParameters = JsonUtils.fromJson(jobParameters.get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), PublishLogicalDatabaseChangeReq.class); sqlRewriter = new RelationFactorRewriter(); executionGroups = new ArrayList<>(); + initExecutorContext(); } - @Override - public boolean start() throws Exception { + private void initExecutorContext() { try { DialectType dialectType = taskParameters.getLogicalDatabaseResp().getDialectType(); DetailLogicalDatabaseResp detailLogicalDatabaseResp = taskParameters.getLogicalDatabaseResp(); @@ -180,6 +181,13 @@ public boolean start() throws Exception { } catch (Exception ex) { log.warn("start logical database change task failed, ", ex); context.getExceptionListener().onException(ex); + } + } + + @Override + public boolean start() throws Exception { + if (Objects.isNull(this.executionGroupContext)) { + log.warn("logical database change task is not initialized"); return false; } while (!Thread.currentThread().isInterrupted()) { From beabc1fb84cfb8b3c3b90eee3fd3d65d5af3ae50 Mon Sep 17 00:00:00 2001 From: guowl3 Date: Fri, 10 Jan 2025 10:20:09 +0800 Subject: [PATCH 096/118] fix(schedule): remove slow sql #4130 --- .../schedule/ScheduleTaskRepositoryTest.java | 12 ---------- .../odc/core/shared/constant/TaskStatus.java | 4 ++++ .../schedule/ScheduleTaskRepository.java | 2 -- .../odc/service/schedule/ScheduleService.java | 5 ++--- .../service/schedule/ScheduleTaskService.java | 22 ------------------- 5 files changed, 6 insertions(+), 39 deletions(-) diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepositoryTest.java index bef16984da..67f59a2142 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepositoryTest.java @@ -17,7 +17,6 @@ import java.util.Collections; import java.util.Date; -import java.util.LinkedList; import java.util.List; import java.util.Optional; @@ -77,17 +76,6 @@ public void updateExecutor() { Assert.equals(executor, byId.get().getExecutor()); } - @Test - public void listByScheduleIdAndStatus() { - taskRepository.deleteAll(); - createScheduleTask(); - List statuses = new LinkedList<>(); - statuses.add(TaskStatus.RUNNING); - statuses.add(TaskStatus.PREPARING); - List byJobNameAndStatus = taskRepository.findByJobNameAndStatusIn("1", statuses); - Assert.equals(byJobNameAndStatus.size(), 1); - } - @Test public void findByIdIn() { ScheduleTaskEntity scheduleTask = createScheduleTask(); diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java index c2290e57e2..6f86cdd5f1 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java @@ -36,6 +36,10 @@ public static List getProcessingStatus() { return Arrays.asList(PREPARING, RUNNING); } + public boolean isProcessing() { + return getProcessingStatus().contains(this); + } + public boolean isTerminated() { return TaskStatus.CANCELED == this || TaskStatus.FAILED == this || TaskStatus.DONE == this; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepository.java index bcd9639d03..f94c21a717 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepository.java @@ -49,8 +49,6 @@ public interface ScheduleTaskRepository extends JpaRepository findByIdAndJobName(Long id, String scheduleId); - List findByJobNameAndStatusIn(String jobName, List statuses); - @Query(value = "select * from schedule_task where job_name=:jobName and job_group = :jobGroup order by id desc limit 1", nativeQuery = true) Optional getLatestScheduleTaskByJobNameAndJobGroup(@Param("jobName") String jobName, diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 6d8f302070..4bc5fca2fd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -718,9 +718,8 @@ public void refreshScheduleStatus(Long scheduleId) { if (status == ScheduleStatus.PAUSE) { return; } - int runningTask = scheduleTaskService.listTaskByJobNameAndStatus(scheduleId.toString(), - TaskStatus.getProcessingStatus()).size(); - if (runningTask > 0) { + Optional latestTask = getLatestTask(scheduleId); + if (latestTask.isPresent() && latestTask.get().getStatus().isProcessing()) { status = ScheduleStatus.ENABLED; } else { try { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java index 9435aaed0d..2f2f1dd7a5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java @@ -284,10 +284,6 @@ public List listByJobNames(Set jobNames) { .collect(Collectors.toList()); } - public List listTaskByJobNameAndStatus(String jobName, List statuses) { - return scheduleTaskRepository.findByJobNameAndStatusIn(jobName, statuses); - } - public ScheduleTask nullSafeGetByJobId(Long jobId) { return findByJobId(jobId) .orElseThrow(() -> new NotFoundException(ResourceType.ODC_SCHEDULE_TASK, "jobId", jobId)); @@ -323,24 +319,6 @@ public ScheduleTask nullSafeGetByIdAndScheduleId(Long id, Long scheduleId) { .orElseThrow(() -> new NotFoundException(ResourceType.ODC_SCHEDULE_TASK, "id", id))); } - - public void correctScheduleTaskStatus(Long scheduleId) { - List toBeCorrectedList = listTaskByJobNameAndStatus( - scheduleId.toString(), TaskStatus.getProcessingStatus()); - // For the scenario where the task framework is switched from closed to open, it is necessary to - // correct - // the status of tasks that were not completed while in the closed state. - if (taskFrameworkEnabledProperties.isEnabled()) { - toBeCorrectedList = - toBeCorrectedList.stream().filter(o -> o.getJobId() == null).collect(Collectors.toList()); - } - toBeCorrectedList.forEach(task -> { - updateStatusById(task.getId(), TaskStatus.CANCELED); - log.info("Task status correction successful,scheduleTaskId={}", task.getId()); - }); - } - - @SkipAuthorize("odc internal usage") public void triggerDataArchiveDelete(Long scheduleTaskId) { From 942f32882ea055d4120072e6cdbba66c1172e7b6 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Fri, 10 Jan 2025 11:24:43 +0800 Subject: [PATCH 097/118] fix(session): kill oracle session may cause sockettimeout #4148 --- .../odc/plugin/connect/oracle/OracleSessionExtension.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java b/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java index 0d0d659a4a..c1dda63316 100644 --- a/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java +++ b/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java @@ -52,7 +52,7 @@ public String getKillQuerySql(@NonNull String connectionId) { @Override public String getKillSessionSql(@NonNull String connectionId) { - return "ALTER SYSTEM KILL SESSION '" + connectionId + "'"; + return "ALTER SYSTEM KILL SESSION '" + connectionId + "'" + " IMMEDIATE"; } @Override From 6aabad1bd3def91c590fa836598867a043262c3f Mon Sep 17 00:00:00 2001 From: guowl3 Date: Fri, 10 Jan 2025 15:42:20 +0800 Subject: [PATCH 098/118] fix(schedule): cherry pick describe database (#4150) * do not request api when describe database (#4147) * fix describe database (#4149) --- .../connection/database/DatabaseService.java | 24 +++++++++++++++++++ .../processor/ScheduleChangePreprocessor.java | 5 +++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index f415778fd5..564cfdc948 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -251,6 +251,12 @@ public Database detailSkipPermissionCheck(@NonNull Long id) { .orElseThrow(() -> new NotFoundException(ResourceType.ODC_DATABASE, "id", id)), true); } + @SkipAuthorize("internal usage") + public Database detailSkipPermissionCheckForRead(@NonNull Long id) { + return entityToModelForRead(databaseRepository.findById(id) + .orElseThrow(() -> new NotFoundException(ResourceType.ODC_DATABASE, "id", id)), true); + } + @SkipAuthorize("odc internal usage") public Database getBasicSkipPermissionCheck(Long id) { return databaseMapper.entityToModel(databaseRepository.findById(id) @@ -984,6 +990,24 @@ private Page entitiesToModels(Page entities, boolean i }); } + private Database entityToModelForRead(DatabaseEntity entity, boolean includesPermittedAction) { + Database model = databaseMapper.entityToModel(entity); + if (Objects.nonNull(entity.getProjectId())) { + model.setProject(projectService.detail(entity.getProjectId())); + } + // for logical database, the connection id may be null + if (entity.getConnectionId() != null) { + model.setDataSource(connectionService.getBasicWithoutPermissionCheck(entity.getConnectionId())); + } + model.setEnvironment(environmentService.detailSkipPermissionCheck(entity.getEnvironmentId())); + if (includesPermittedAction) { + model.setAuthorizedPermissionTypes( + permissionHelper.getDBPermissions(Collections.singleton(entity.getId())).get(entity.getId())); + } + return model; + } + + private Database entityToModel(DatabaseEntity entity, boolean includesPermittedAction) { Database model = databaseMapper.entityToModel(entity); if (Objects.nonNull(entity.getProjectId())) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java index 43359d91cc..729895ebe8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java @@ -25,6 +25,7 @@ import org.springframework.stereotype.Component; import com.oceanbase.odc.core.shared.exception.UnsupportedException; +import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.dlm.model.DataArchiveParameters; @@ -53,6 +54,8 @@ public class ScheduleChangePreprocessor implements InitializingBean { @Autowired private DatabaseService databaseService; @Autowired + private ConnectionService connectionService; + @Autowired private ScheduleService scheduleService; @Autowired private List preprocessors; @@ -94,7 +97,7 @@ public void afterPropertiesSet() throws Exception { } private void adaptScheduleChangeParams(ScheduleChangeParams req) { - Database srcDb = databaseService.detail(getTargetDatabaseId(req)); + Database srcDb = databaseService.detailSkipPermissionCheckForRead(getTargetDatabaseId(req)); req.setProjectId(srcDb.getProject().getId()); req.setProjectName(srcDb.getProject().getName()); req.setConnectionId(srcDb.getDataSource().getId()); From 34667967db8f73fd8695568945c2120fae797787 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Fri, 10 Jan 2025 18:21:20 +0800 Subject: [PATCH 099/118] fix(session): kill session may happen invalid server ip address #4144 * IP REGEX outdate * remove useless code --- .../odc/service/db/session/DefaultDBSessionManage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index e609a56150..177af4215e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -82,8 +82,8 @@ @Slf4j public class DefaultDBSessionManage implements DBSessionManageFacade { - private static final String SERVER_REGEX = ".*/\\*(?([0-9]{1,3}.){1,3}([0-9]{1,3})):" - + "(?[0-9]{1,5})\\*/.*"; + private static final String SERVER_REGEX = ".*(?([0-9]{1,3}.){1,3}([0-9]{1,3})):" + + "(?[0-9]{1,5}).*"; private static final Pattern SERVER_PATTERN = Pattern.compile(SERVER_REGEX); private static final ConnectionMapper CONNECTION_MAPPER = ConnectionMapper.INSTANCE; private static final String GLOBAL_CLIENT_SESSION_OB_PROXY_VERSION_NUMBER = "4.2.3"; From f4f03439ac08e39d3f3060e8ceb5196099a94d78 Mon Sep 17 00:00:00 2001 From: "zijia.cj" Date: Mon, 13 Jan 2025 10:07:40 +0800 Subject: [PATCH 100/118] fix(sql check):add two sql check rules for CREATE LIKE/AS statement * Add checks for LIKE table in MySQLNoTableCommentExists and NoPrimaryKeyExists rules * Add SQL check rules for CREATE LIKE/AS table statements * Optimize CREATE LIKE statement checks and rule descriptions * Update CREATE LIKE/AS rule descriptions and names * Refactor SQL check rules for CREATE LIKE/AS conditions * Fix punctuation in SQL check messages and disable CREATE LIKE/AS rules * Enable SQL check rules in regulation-rule-applying.yaml * Fix SQL check rule descriptions in regulation metadata * Fix SQL check rule descriptions in regulation metadata * Fix SQL check rule descriptions in regulation metadata --- .../i18n/BusinessMessages.properties | 206 +++++++++--------- .../i18n/BusinessMessages_zh_CN.properties | 8 + .../i18n/BusinessMessages_zh_TW.properties | 8 + .../init/regulation-rule-applying.yaml | 80 +++++++ .../init/regulation-rule-metadata.yaml | 38 +++- .../factory/CreateTableAsExistsFactory.java | 44 ++++ .../factory/CreateTableLikeExistsFactory.java | 44 ++++ .../sqlcheck/model/SqlCheckRuleType.java | 11 +- .../rule/BaseMissingRequiredColumns.java | 7 +- .../sqlcheck/rule/CreateTableAsExists.java | 64 ++++++ .../sqlcheck/rule/CreateTableLikeExists.java | 63 ++++++ .../rule/MySQLNoTableCommentExists.java | 4 + .../rule/MySQLRestrictTableAutoIncrement.java | 4 + .../sqlcheck/rule/NoPrimaryKeyExists.java | 4 + .../rule/OracleNoTableCommentExists.java | 7 +- .../service/sqlcheck/rule/SqlCheckRules.java | 4 + 16 files changed, 493 insertions(+), 103 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableAsExistsFactory.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableLikeExistsFactory.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableAsExists.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableLikeExists.java diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties index e7d335972b..c43c144c4b 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties @@ -508,28 +508,28 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-inte # # sql-check module # -com.oceanbase.odc.CheckViolation.LocalizedMessage=There may be a problem with statement at line {0}, col {1}, detail: {2} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.message=There is a syntax error in the statement, details: {0} +com.oceanbase.odc.CheckViolation.LocalizedMessage=There may be a problem with statement at line {0}, col {1}, detail: {2}. +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.message=There is a syntax error in the statement, details: {0}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.name=Syntax Error -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.description=The statement to be checked has a syntax error +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.description=The statement to be checked has a syntax error. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.message=Calculation on the column condition, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.message=Calculation on the column condition, which may cause the index to fail. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.name=Calculation on the column condition -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.description=Calculation on the column condition, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.description=Calculation on the column condition, which may cause the index to fail. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.message=There is a left fuzzy match in the LIKE operation, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.message=There is a left fuzzy match in the LIKE operation, which may cause the index to fail. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.name=Left fuzzy match in the LIKE operation -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.description=There is a left fuzzy match in the LIKE operation, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.description=There is a left fuzzy match in the LIKE operation, which may cause the index to fail. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.message=Implicit type conversion on the column condition, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.message=Implicit type conversion on the column condition, which may cause the index to fail. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.name=Implicit type conversion -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.description=Implicit type conversion on the column condition, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.description=Implicit type conversion on the column condition, which may cause the index to fail. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.message=Too many IN values to match may reduce the query efficiency, it is recommended not to exceed {0}, the actual is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.message=Too many IN values to match may reduce the query efficiency, it is recommended not to exceed {0}, the actual is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.name=Too many IN values to match -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.description=Too many IN values to match may reduce the query efficiency +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.description=Too many IN values to match may reduce the query efficiency. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count=Maximum number of IN expressions -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count.description=The maximum number of expressions in the IN operation in the conditional predicate +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count.description=The maximum number of expressions in the IN operation in the conditional predicate. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.message=The number of rows affected by this SQL statement exceeds the limit. Maximum Allowed Affected Rows: {0}; Estimated Affected Rows: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.name=Estimated SQL Affected Rows @@ -538,112 +538,112 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affect com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count.description=Maximum Allowed SQL Affected Rows com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.estimate-sql-affected-rows-failed.message=The number of rows affected by this SQL statement cannot be determined or failed to be determined. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.estimate-sql-affected-rows-failed.name=Unable to determine or failed to determine the number of rows affected by the SQL statement. +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.estimate-sql-affected-rows-failed.name=Unable to determine or failed to determine the number of rows affected by the SQL statement com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.estimate-sql-affected-rows-failed.description=Unable to determine or failed to determine the number of rows affected by the SQL statement. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.message=Too many table objects JOIN may lead to an unoptimized execution plan. It is recommended not to exceed {0}, the actual is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.message=Too many table objects JOIN may lead to an unoptimized execution plan. It is recommended not to exceed {0}, the actual is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.name=Too many table objects JOIN -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.description=Too many table objects JOIN may lead to an unoptimized execution plan +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.description=Too many table objects JOIN may lead to an unoptimized execution plan. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count=Max join table count com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count.description=Maximum number of table joins -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.message=NOT NULL is required in the NOT IN condition to avoid falling into nested loops +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.message=NOT NULL is required in the NOT IN condition to avoid falling into nested loops. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.name=NOT NULL is required in the NOT IN condition -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.description=NOT NULL is required in the NOT IN condition to avoid falling into nested loops +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.description=NOT NULL is required in the NOT IN condition to avoid falling into nested loops. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.message=Potential risk of constant true/false WHERE conditions in UPDATE/DELETE statements com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.name=No valid WHERE condition in UPDATE/DELETE statement com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.description=Potential risk of constant true/false WHERE conditions in UPDATE/DELETE statements -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.message=No WHERE condition in UPDATE/DELETE statement poses potential risk +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.message=No WHERE condition in UPDATE/DELETE statement poses potential risk. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.name=No WHERE condition in UPDATE/DELETE statement -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.description=No WHERE condition in UPDATE/DELETE statement poses potential risk +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.description=No WHERE condition in UPDATE/DELETE statement poses potential risk. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.message=INSERT/REPLACE statements without specific columns may cause implicit conversion errors +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.message=INSERT/REPLACE statements without specific columns may cause implicit conversion errors. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.name=INSERT/REPLACE statement does not specify column names -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.description=INSERT/REPLACE statements without specific columns may cause implicit conversion errors +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.description=INSERT/REPLACE statements without specific columns may cause implicit conversion errors. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.message=Operation on table object {0} with index does not use index, may cause full table scan, resulting in performance degradation +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.message=Operation on table object {0} with index does not use index, may cause full table scan, resulting in performance degradation. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.name=No index exists -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.description=Operation on table object with index does not use index, may cause full table scan, resulting in performance degradation +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.description=Operation on table object with index does not use index, may cause full table scan, resulting in performance degradation. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.message=Operation on partitioned table {0} does not use a partition key, resulting in partition pruning not possible +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.message=Operation on partitioned table {0} does not use a partition key, resulting in partition pruning not possible. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.name=No partition exists -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.description=Operation on partitioned table does not use a partition key, resulting in partition pruning not possible +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.description=Operation on partitioned table does not use a partition key, resulting in partition pruning not possible. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.message=There are too many indexes on the table object {0}, which may cause performance degradation, it is recommended not to exceed {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.message=There are too many indexes on the table object {0}, which may cause performance degradation, it is recommended not to exceed {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.name=Single table contains too many indexes -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.description=Using too many indexes may cause performance degradation +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.description=Using too many indexes may cause performance degradation. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count=Max index count -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count.description=The maximum number of indexes that can appear in a table definition +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count.description=The maximum number of indexes that can appear in a table definition. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.message=It is recommended to use local indexes +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.message=It is recommended to use local indexes. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.name=Perfer local indexes -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.description=It is recommended to use local indexes +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.description=It is recommended to use local indexes. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.message=Table object {0} has too many columns, recommended no more than {1}, actual {2} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.message=Table object {0} has too many columns, recommended no more than {1}, actual {2}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.name=Single table contains too many columns -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.description=Single table contains too many columns +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.description=Single table contains too many columns. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count=Max column definition count -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count.description=The maximum number of columns that can appear in a table definition +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count.description=The maximum number of columns that can appear in a table definition. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.message=The length of the char type in the table definition is too long, it is recommended not to exceed {0}, the actual value is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.message=The length of the char type in the table definition is too long, it is recommended not to exceed {0}, the actual value is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.name=Limit the length of char type fields -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.description=The length of the char type field should not be too long +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.description=The length of the char type field should not be too long. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length=char maximum allowed length com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length.description=char maximum allowed length -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.message=The named {0} format of the unique index does not conform to the specification, pattern is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.message=The named {0} format of the unique index does not conform to the specification, pattern is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name=Restrict unique index name format com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.description=The standardized naming of unique indexes helps to improve the efficiency of database development, default is uk_${table-name}_${column-name-1}_${column-name-2}_... com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern=Unique index name regular expression com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern.description=Unique index name regular expression -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.message=Tables cannot use foreign keys +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.message=Tables cannot use foreign keys. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.name=Tables cannot use foreign keys -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.description=Tables cannot use foreign keys +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.description=Tables cannot use foreign keys. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.message=The table must have a primary key +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.message=The table must have a primary key. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.name=The table must have a primary key -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.description=The table must have a primary key +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.description=The table must have a primary key. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.message=The creation statement of the table object {0} must contain comments +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.message=The creation statement of the table object {0} must contain comments. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.name=Table must be commented -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.description=Table object existence annotation can help to quickly understand the business background of the table +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.description=Table object existence annotation can help to quickly understand the business background of the table. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.message=Table object cannot be named with {0}, the name is blacklisted at {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.message=Table object cannot be named with {0}, the name is blacklisted at {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.name=Table names cannot be in the blacklist -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.description=The table name when creating the table, and the table name in the blacklist cannot be used when modifying the table name +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.description=The table name when creating the table, and the table name in the blacklist cannot be used when modifying the table name. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list=Black list com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list.description=Table Name blacklist -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.message=The character set {0} used by the table object is not in the allowed range. Allowed character sets include: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.message=The character set {0} used by the table object is not in the allowed range. Allowed character sets include: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.name=Limit Table Character Set -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.description=The charset used by the table object can only be selected from the options given in the whitelist +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.description=The charset used by the table object can only be selected from the options given in the whitelist. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets=Allowed character sets com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets.description=Allowed character sets -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.message=The collation {0} used by the table object is not in the allowed range. Allowed collations include: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.message=The collation {0} used by the table object is not in the allowed range. Allowed collations include: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.name=Restricts the collation of table objects -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.description=The collation used by the table object can only be selected from the options given in the whitelist +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.description=The collation used by the table object can only be selected from the options given in the whitelist. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations=Allowed collations com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations.description=Allowed collations -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.message=There are {0} column references in the index out of a maximum of {1} column references +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.message=There are {0} column references in the index out of a maximum of {1} column references. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.name=Limit the number of columns included in a single index -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.description=The number of column references in a single index definition should not be excessive +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.description=The number of column references in a single index definition should not be excessive. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count=Maximum number of column references com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count.description=Maximum number of column references -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.message=The primary key constraint involves a column whose type {0} is not allowed, allowed types include: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.message=The primary key constraint involves a column whose type {0} is not allowed, allowed types include: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.name=Restrict primary key data type -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.description=The columns involved in the primary key of the table must be of the data type defined in the rule +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.description=The columns involved in the primary key of the table must be of the data type defined in the rule. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes=Types allowed as primary keys com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes.description=Types allowed as primary keys -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.message=There are {0} column references in the primary key out of a maximum of {1} column references +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.message=There are {0} column references in the primary key out of a maximum of {1} column references. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.name=Limit the number of columns included in primary key -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.description=The number of column references in primary key definition should not be excessive +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.description=The number of column references in primary key definition should not be excessive. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count=Maximum number of column references com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count.description=Maximum number of column references @@ -651,51 +651,51 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-in com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.name=Restrict primary key columns to auto-increment com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.description=Restrict primary key columns to auto-increment -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.message=The named {0} format of the index does not conform to the specification, pattern is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.message=The named {0} format of the index does not conform to the specification, pattern is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name=Restrict index name format com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.description=The standardized naming of indexes helps to improve the efficiency of database development, default is idx_${table-name}_${column-name-1}_${column-name-2}_... com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern=Index name regular expression com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern.description=Index name regular expression -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.message=The definition of an index or constraint is not named +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.message=The definition of an index or constraint is not named. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.name=Index or constraint needs to set name -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.description=An index or constraint requires an explicitly defined name, otherwise the database names it automatically +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.description=An index or constraint requires an explicitly defined name, otherwise the database names it automatically. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.message=Numeric types use the ZEROFILL attribute +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.message=Numeric types use the ZEROFILL attribute. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.name=Numeric types cannot use the ZEROFILL attribute -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.description=Numeric types cannot use the ZEROFILL attribute +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.description=Numeric types cannot use the ZEROFILL attribute. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.message=Cannot set character set on column +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.message=Cannot set character set on column. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.name=Cannot set character set on column -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.description=Cannot set character set on column +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.description=Cannot set character set on column. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.message=Cannot set collation on column +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.message=Cannot set collation on column. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.name=Cannot set collation on column -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.description=Cannot set collation on column +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.description=Cannot set collation on column. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.message=Columns of type {0} are not allowed to be null, and the data types of columns that are allowed to be null is: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.message=Columns of type {0} are not allowed to be null, and the data types of columns that are allowed to be null is: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.name=Restrict column not nullable (NOT NULL) com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.description=Restrict columns to be non-nullable com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list=Nullable column data types com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list.description=Nullable column data types -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.message=Column with datatype {0} is not allowed without default value, column types without default are allowed: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.message=Column with datatype {0} is not allowed without default value, column types without default are allowed: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.name=Column has a default value -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.description=Column has a default value +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.description=Column has a default value. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list=Types without default values are allowed -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list.description=Types without default values are allowed +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list.description=Types without default values are allowed. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.message=Column {0} has no column comments +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.message=Column {0} has no column comments. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.name=Column to be commented -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.description=Each column in the table definition needs to be commented +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.description=Each column in the table definition needs to be commented. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.message=Column cannot be named with {0}, the name is blacklisted at {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.message=Column cannot be named with {0}, the name is blacklisted at {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.name=Column names cannot be in the blacklist -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.description=The column name when creating the table, and the column name in the blacklist cannot be used when modifying the column name +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.description=The column name when creating the table, and the column name in the blacklist cannot be used when modifying the column name. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list=Black list com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list.description=Column name blacklist -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.message=The case setting of the column name {0} does not conform to the specification +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.message=The case setting of the column name {0} does not conform to the specification. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name=Restrict column name case com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.description=Restrict column name case com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase=Whether to limit uppercase @@ -703,7 +703,7 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-nam com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase=Whether to limit lowercase com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase.description=Whether to limit lowercase -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.message=The case setting of the table name {0} does not conform to the specification +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.message=The case setting of the table name {0} does not conform to the specification. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name=Restrict table name case com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.description=Restrict table name case com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase=Whether to limit uppercase @@ -711,82 +711,90 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase=Whether to limit lowercase com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase.description=Whether to limit lowercase -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.message=The initial value of the auto-increment column of the restricted table is {0}, but the actual value is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.message=The initial value of the auto-increment column of the restricted table is {0}, but the actual value is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.name=Restrict auto-increment initial value for table creation com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.description=Restrict auto-increment initial value for table creation com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value=Table auto-increment initial value com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value.description=Table auto-increment initial value -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.message=It is not recommended to use * to project all columns in the SELECT statement +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.message=It is not recommended to use * to project all columns in the SELECT statement. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.name=SELECT statement is deprecated * -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.description=SELECT statement is deprecated * +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.description=SELECT statement is deprecated *. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.message=Table is missing required column: {0} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.message=Table is missing required column: {0}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.name=Table is missing required column -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.description=Table is missing required column +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.description=Table is missing required column. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names=Collection of column names com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names.description=Collection of column names -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.message=Auto-increment column {0} is better to use unsigned type +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.message=Auto-increment column {0} is better to use unsigned type. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.name=Restrict auto-increment columns to use UNSIGNED type -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.description=The unsigned type does not store negative numbers, and the value range stored in the same type is doubled, which can prevent the auto-increment column from exceeding the upper limit +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.description=The unsigned type does not store negative numbers, and the value range stored in the same type is doubled, which can prevent the auto-increment column from exceeding the upper limit. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.message=There are {0} modify statements for the same table {1}, exceeding the maximum limit of {2}, it is recommended to combine them into one ALTER statement +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.message=There are {0} modify statements for the same table {1}, exceeding the maximum limit of {2}, it is recommended to combine them into one ALTER statement. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.name=Limit the number of ALTERs executed on the same table at one time -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.description=Avoid the consumption caused by multiple table rebuilds and the impact on online business +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.description=Avoid the consumption caused by multiple table rebuilds and the impact on online business. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count=Maximum number of ALTERs -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count.description=The maximum number of ALTERs allowed for the same table +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count.description=The maximum number of ALTERs allowed for the same table. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.message=Columns are marked as NOT NULL but do not have a default value set, which may result in insert errors +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.message=Columns are marked as NOT NULL but do not have a default value set, which may result in insert errors. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.name=When the field constraint is NOT NULL, it must have a default value -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.description=When the field constraint is NOT NULL, it must have a default value +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.description=When the field constraint is NOT NULL, it must have a default value. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.message=The named {0} format of the primary key does not conform to the specification, pattern is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.message=The named {0} format of the primary key does not conform to the specification, pattern is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name=Restrict primary key name format com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.description=The standardized naming of PK helps to improve the efficiency of database development, default is pk_${table-name}_${column-name-1}_${column-name-2}_... com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern=PK name regular expression com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern.description=PK name regular expression -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.message=Datatype {0} is forbidden, forbidden datatypes: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.message=Datatype {0} is forbidden, forbidden datatypes: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.name=Certain data types are prohibited -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.description=Certain data types are prohibited +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.description=Certain data types are prohibited. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names=Prohibited data types com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names.description=Prohibited data types -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.message=Index reference column {0} is of datatype {1}, which is not allowed to be indexed, allowed types: {2} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.message=Index reference column {0} is of datatype {1}, which is not allowed to be indexed, allowed types: {2}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.name=Restricted Index Data Types -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.description=Only columns of a specific data type can be referenced by the index, avoiding a large number of resources caused by incorrect indexing and serious performance problems +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.description=Only columns of a specific data type can be referenced by the index, avoiding a large number of resources caused by incorrect indexing and serious performance problems. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes=Allowed data types com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes.description=Data types allowed in indexes -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.message=The database object type {0} involved in the object delete statement is not within the allowed range, the object types allowed to be deleted include: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.message=The database object type {0} involved in the object delete statement is not within the allowed range, the object types allowed to be deleted include: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.name=Limit the types of objects that can be deleted -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.description=Deletion of database objects outside the allowed range is not allowed +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.description=Deletion of database objects outside the allowed range is not allowed. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types=Database object type that allowed to be deleted -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types.description=The types of database objects that are allowed to be deleted +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types.description=The types of database objects that are allowed to be deleted. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.message=Primary key constraint/index are not named +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.message=Primary key constraint/index are not named. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.name=Primary key constraint/index need to set names -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.description=Primary key constraint/index need to be explicitly named, otherwise the database will automatically name them +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.description=Primary key constraint/index need to be explicitly named, otherwise the database will automatically name them. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.message=Column {0} is marked as auto-increment. The column type is {1}, which is not within the allowed type range: {2}. This may lead to unexpected situations such as numeric overflow +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.message=Column {0} is marked as auto-increment. The column type is {1}, which is not within the allowed type range: {2}. This may lead to unexpected situations such as numeric overflow. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.name=Limit optional data types for auto-increment columns -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.description=The data type of columns marked as auto-increment should be carefully selected to avoid unexpected situations such as numerical overflow +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.description=The data type of columns marked as auto-increment should be carefully selected to avoid unexpected situations such as numerical overflow. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.allowed-datatypes=Allowed data types com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.allowed-datatypes.description=A collection of data types that allow columns to be marked as auto-increment com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.message={0} is a reserved word and should not be used as an object name. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.name=Reserved words should not be used as object names -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.description=Reserved words should not be used as object names +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.description=Reserved words should not be used as object names. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.message={0} is an offline structure change statement, which may take a long time. It is recommended to use lock-free structure change to execute this statement. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.name=Offline structure change statement exists com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.description=The offline structure change statement may involve full data modification, which will take a long time and may affect online business. It is recommended to use lock-free structure change to perform this operation. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.message=The truncate statement will clear the table data, please use it with caution +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.message=The truncate statement will clear the table data, please use it with caution. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.name=Truncate statement exists com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.description=The truncate statement will clear table data and is very dangerous in a production environment. Please use it with caution. +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.message=It is not recommended to use CREATE LIKE statement to create a table. +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name=CREATE LIKE statement exists +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.description=It is not recommended to use CREATE LIKE statement to create a table. + +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.message=It is not recommended to use CREATE AS statement to create a table. +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name=CREATE AS statement exists +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.description=It is not recommended to use CREATE AS statement to create a table. + # below masking algorithm built-in com.oceanbase.odc.builtin-resource.masking-algorithm.mask-all.name=Mask all (system default) com.oceanbase.odc.builtin-resource.masking-algorithm.personal-name-chinese.name=Personal name (Chinese character) diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties index 280ff945ba..dbe6d0d79b 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties @@ -726,6 +726,14 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exis com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.name=存在 truncate 语句 com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.description=truncate 语句将会清空表数据，在生产环境中十分危险，请谨慎使用 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.message=不建议使用 CREATE LIKE 语句建表 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name=存在 CREATE LIKE 语句 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.description=不建议使用 CREATE LIKE 语句建表 + +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.message=不建议使用 CREATE AS 语句建表 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name=存在 CREATE AS 语句 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.description=不建议使用 CREATE AS 语句建表 + # below masking algorithm built-in com.oceanbase.odc.builtin-resource.masking-algorithm.mask-all.name=全部遮掩（系统默认） com.oceanbase.odc.builtin-resource.masking-algorithm.personal-name-chinese.name=个人姓名（汉字类型） diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties index 93555afddc..6f3b3a207e 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties @@ -791,6 +791,14 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exis com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.name=存在 truncate 語句 com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.description=truncate 語句將會清空表格數據，在生產環境中十分危險，請謹慎使用 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.message=不建議使用 CREATE LIKE 語句建表 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name=存在 CREATE LIKE 語句 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.description=不建議使用 CREATE LIKE 語句建表 + +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.message=不建議使用 CREATE AS 語句建表 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name=存在 CREATE AS 語句 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.description=不建議使用 CREATE AS 語句建表 + # below masking algorithm built-in com.oceanbase.odc.builtin-resource.masking-algorithm.mask-all.name=全部遮掩（系統默認） com.oceanbase.odc.builtin-resource.masking-algorithm.personal-name-chinese.name=個人姓名（漢字類型） diff --git a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml index 9f58372854..1db76776b7 100644 --- a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml +++ b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml @@ -419,6 +419,26 @@ - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-sit-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-sit-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'OB_ORACLE' + - 'ORACLE' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' - enabled: 1 level: 0 rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-sit-ruleset.name} @@ -964,6 +984,26 @@ - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-dev-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-dev-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'OB_ORACLE' + - 'ORACLE' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' - enabled: 1 level: 0 rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-dev-ruleset.name} @@ -1509,6 +1549,26 @@ - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-prod-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-prod-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'OB_ORACLE' + - 'ORACLE' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' - enabled: 1 level: 1 rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-prod-ruleset.name} @@ -2043,6 +2103,26 @@ - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-default-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-default-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'OB_ORACLE' + - 'ORACLE' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' - enabled: 1 level: 1 rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-default-ruleset.name} diff --git a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml index e9d18e970d..c9a686d128 100644 --- a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml +++ b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml @@ -1552,4 +1552,40 @@ - label: SUPPORTED_DIALECT_TYPE value: OB_MYSQL - label: SUPPORTED_DIALECT_TYPE - value: MYSQL \ No newline at end of file + value: MYSQL +- id: 61 + name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name} + description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.description} + type: SQL_CHECK + builtIn: 1 + labels: + - label: SUB_TYPE + value: DDL + - label: SUB_TYPE + value: TABLE + - label: SUPPORTED_DIALECT_TYPE + value: OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ODP_SHARDING_OB_MYSQL +- id: 62 + name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name} + description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.description} + type: SQL_CHECK + builtIn: 1 + labels: + - label: SUB_TYPE + value: DDL + - label: SUB_TYPE + value: TABLE + - label: SUPPORTED_DIALECT_TYPE + value: MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + - label: SUPPORTED_DIALECT_TYPE + value: OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: OB_ORACLE + - label: SUPPORTED_DIALECT_TYPE + value: ODP_SHARDING_OB_MYSQL diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableAsExistsFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableAsExistsFactory.java new file mode 100644 index 0000000000..1f58504347 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableAsExistsFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.sqlcheck.factory; + +import java.util.Map; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRuleFactory; +import com.oceanbase.odc.service.sqlcheck.model.SqlCheckRuleType; +import com.oceanbase.odc.service.sqlcheck.rule.CreateTableAsExists; + +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2025/1/8 20:33 + * @since: 4.3.3 + */ +public class CreateTableAsExistsFactory implements SqlCheckRuleFactory { + @Override + public SqlCheckRuleType getSupportsType() { + return SqlCheckRuleType.CREATE_TABLE_AS_EXISTS; + } + + @Override + public SqlCheckRule generate(@NonNull DialectType dialectType, Map parameters) { + return new CreateTableAsExists(); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableLikeExistsFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableLikeExistsFactory.java new file mode 100644 index 0000000000..2a783209a6 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableLikeExistsFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.sqlcheck.factory; + +import java.util.Map; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRuleFactory; +import com.oceanbase.odc.service.sqlcheck.model.SqlCheckRuleType; +import com.oceanbase.odc.service.sqlcheck.rule.CreateTableLikeExists; + +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2025/1/8 18:41 + * @since: 4.3.3 + */ +public class CreateTableLikeExistsFactory implements SqlCheckRuleFactory { + @Override + public SqlCheckRuleType getSupportsType() { + return SqlCheckRuleType.CREATE_TABLE_LIKE_EXISTS; + } + + @Override + public SqlCheckRule generate(@NonNull DialectType dialectType, Map parameters) { + return new CreateTableLikeExists(); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java index 752558df8f..53850924bf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java @@ -251,7 +251,16 @@ public enum SqlCheckRuleType implements Translatable { /** * Unable to judge restrict the number of lines affected by SQL */ - ESTIMATE_SQL_AFFECTED_ROWS_FAILED("estimate-sql-affected-rows-failed"); + ESTIMATE_SQL_AFFECTED_ROWS_FAILED("estimate-sql-affected-rows-failed"), + /** + * Create like statement exists + */ + CREATE_TABLE_LIKE_EXISTS("create-table-like-exists"), + /** + * Create as statement exists + */ + CREATE_TABLE_AS_EXISTS("create-table-as-exists"); + private final String name; private static final String NAME_CODE = "name"; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseMissingRequiredColumns.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseMissingRequiredColumns.java index d415c58f86..3a25d2515e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseMissingRequiredColumns.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseMissingRequiredColumns.java @@ -18,6 +18,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -56,7 +57,11 @@ public List check(@NonNull Statement statement, @NonNull SqlChec if (!(statement instanceof CreateTable)) { return Collections.emptyList(); } - List columns = ((CreateTable) statement).getColumnDefinitions().stream() + CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable()) || Objects.nonNull(createTable.getAs())) { + return Collections.emptyList(); + } + List columns = createTable.getColumnDefinitions().stream() .map(d -> unquoteIdentifier(d.getColumnReference().getColumn())).collect(Collectors.toList()); Set tmp = new HashSet<>(requiredColumns); tmp.removeIf(s -> columns.contains(unquoteIdentifier(s))); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableAsExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableAsExists.java new file mode 100644 index 0000000000..815a1006de --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableAsExists.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.sqlcheck.rule; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.service.sqlcheck.SqlCheckContext; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; +import com.oceanbase.odc.service.sqlcheck.SqlCheckUtil; +import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; +import com.oceanbase.odc.service.sqlcheck.model.SqlCheckRuleType; +import com.oceanbase.tools.sqlparser.statement.Statement; +import com.oceanbase.tools.sqlparser.statement.createtable.CreateTable; + +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2025/1/8 17:34 + * @since: 4.3.3 + */ +public class CreateTableAsExists implements SqlCheckRule { + @Override + public SqlCheckRuleType getType() { + return SqlCheckRuleType.CREATE_TABLE_AS_EXISTS; + } + + @Override + public List check(@NonNull Statement statement, @NonNull SqlCheckContext context) { + if (!(statement instanceof CreateTable)) { + return Collections.emptyList(); + } + CreateTable createTable = (CreateTable) statement; + if ((Objects.nonNull(createTable.getAs()))) { + return Collections.singletonList(SqlCheckUtil.buildViolation( + statement.getText(), statement, getType(), new Object[] {})); + } + return Collections.emptyList(); + } + + @Override + public List getSupportsDialectTypes() { + return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableLikeExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableLikeExists.java new file mode 100644 index 0000000000..709b5455b9 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableLikeExists.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.sqlcheck.rule; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.service.sqlcheck.SqlCheckContext; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; +import com.oceanbase.odc.service.sqlcheck.SqlCheckUtil; +import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; +import com.oceanbase.odc.service.sqlcheck.model.SqlCheckRuleType; +import com.oceanbase.tools.sqlparser.statement.Statement; +import com.oceanbase.tools.sqlparser.statement.createtable.CreateTable; + +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2025/1/8 17:31 + * @since: 4.3.3 + */ +public class CreateTableLikeExists implements SqlCheckRule { + @Override + public SqlCheckRuleType getType() { + return SqlCheckRuleType.CREATE_TABLE_LIKE_EXISTS; + } + + @Override + public List check(@NonNull Statement statement, @NonNull SqlCheckContext context) { + if (!(statement instanceof CreateTable)) { + return Collections.emptyList(); + } + CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable())) { + return Collections.singletonList(SqlCheckUtil.buildViolation( + statement.getText(), statement, getType(), new Object[] {})); + } + return Collections.emptyList(); + } + + @Override + public List getSupportsDialectTypes() { + return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.ODP_SHARDING_OB_MYSQL); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLNoTableCommentExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLNoTableCommentExists.java index 69ee5269fc..3a3ca6e943 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLNoTableCommentExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLNoTableCommentExists.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.service.sqlcheck.SqlCheckContext; @@ -51,6 +52,9 @@ public List check(@NonNull Statement statement, @NonNull SqlChec return Collections.emptyList(); } CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable()) || Objects.nonNull(createTable.getAs())) { + return Collections.emptyList(); + } TableOptions tableOptions = createTable.getTableOptions(); if (tableOptions == null || tableOptions.getComment() == null) { return Collections.singletonList(SqlCheckUtil.buildViolation( diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLRestrictTableAutoIncrement.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLRestrictTableAutoIncrement.java index 82b04eb86e..fe49dfaa34 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLRestrictTableAutoIncrement.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLRestrictTableAutoIncrement.java @@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.service.sqlcheck.SqlCheckContext; @@ -58,6 +59,9 @@ public List check(@NonNull Statement statement, @NonNull SqlChec return Collections.emptyList(); } CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable()) || Objects.nonNull(createTable.getAs())) { + return Collections.emptyList(); + } TableOptions options = createTable.getTableOptions(); if (options == null || options.getAutoIncrement() == null) { return Collections.singletonList(SqlCheckUtil.buildViolation( diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java index be1071dbfa..db1f3127e9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import org.apache.commons.collections4.CollectionUtils; @@ -54,6 +55,9 @@ public List check(@NonNull Statement statement, @NonNull SqlChec return Collections.emptyList(); } CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable()) || Objects.nonNull(createTable.getAs())) { + return Collections.emptyList(); + } boolean containsPk = createTable.getColumnDefinitions().stream() .filter(c -> c.getColumnAttributes() != null && CollectionUtils.isNotEmpty(c.getColumnAttributes().getConstraints())) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java index 10ef6ca25e..b3d9ee6d6e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -67,7 +68,11 @@ public List check(@NonNull Statement statement, @NonNull SqlChec if (statement instanceof SetComment) { setComments.add(new Pair<>((SetComment) statement, null)); } else if (statement instanceof CreateTable) { - createTables.add(new Pair<>((CreateTable) statement, null)); + CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable()) || Objects.nonNull(createTable.getAs())) { + return Collections.emptyList(); + } + createTables.add(new Pair<>(createTable, null)); } Set tblNames = setComments.stream().filter(s -> s.left.getTable() != null).map(s -> { RelationFactor t = s.left.getTable(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SqlCheckRules.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SqlCheckRules.java index 1bb755105b..03ab0de64a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SqlCheckRules.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SqlCheckRules.java @@ -39,6 +39,8 @@ import com.oceanbase.odc.service.sqlcheck.factory.ColumnCharsetExistsFactory; import com.oceanbase.odc.service.sqlcheck.factory.ColumnCollationExistsFactory; import com.oceanbase.odc.service.sqlcheck.factory.ColumnNameInBlackListFactory; +import com.oceanbase.odc.service.sqlcheck.factory.CreateTableAsExistsFactory; +import com.oceanbase.odc.service.sqlcheck.factory.CreateTableLikeExistsFactory; import com.oceanbase.odc.service.sqlcheck.factory.ForeignConstraintExistsFactory; import com.oceanbase.odc.service.sqlcheck.factory.LeftFuzzyMatchFactory; import com.oceanbase.odc.service.sqlcheck.factory.MissingRequiredColumnsFactory; @@ -155,6 +157,8 @@ public static List getAllFactories(DialectType dialectType, rules.add(new TruncateTableExistsFactory()); rules.add(new SqlAffectedRowsFactory(jdbc)); rules.add(new Unable2JudgeAffectedRowsFactory(jdbc)); + rules.add(new CreateTableLikeExistsFactory()); + rules.add(new CreateTableAsExistsFactory()); return rules; } From 262c24d9bbcee8051c4cba8f0b2f4009c29ec9ea Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Mon, 13 Jan 2025 10:09:05 +0800 Subject: [PATCH 101/118] security: add @SkipAuthorize in public method #4153 --- .../odc/service/collaboration/project/ProjectService.java | 1 + .../oceanbase/odc/service/connection/ConnectionService.java | 1 + .../odc/service/connection/database/DatabaseService.java | 1 + .../odc/service/connection/table/TableService.java | 1 + .../odc/service/db/session/DefaultDBSessionManage.java | 1 + .../odc/service/iam/GlobalResourceRoleService.java | 6 +++++- .../com/oceanbase/odc/service/iam/ResourceRoleService.java | 1 + .../java/com/oceanbase/odc/service/iam/UserService.java | 2 ++ .../odc/service/integration/IntegrationService.java | 1 + .../odc/service/integration/oauth2/TestLoginManager.java | 2 ++ .../odc/service/iam/auth/saml/DefaultSamlUserService.java | 2 ++ 11 files changed, 18 insertions(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java index a03711a8c5..eff4ab5f7e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java @@ -562,6 +562,7 @@ public void checkCurrentUserProjectRolePermissions(@NotNull Project project, projectPermissionValidator.checkProjectRole(project.getId(), roleNames); } + @SkipAuthorize("inside method permission check") public void checkUnfinishedTickets(@NonNull Long projectId) { if (flowInstanceService.listUnfinishedFlowInstances(Pageable.unpaged(), projectId).hasContent()) { throw new BadRequestException( diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java index 60dcb7b5d1..4900d717c5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java @@ -605,6 +605,7 @@ public ConnectionConfig update(@NotNull Long id, @NotNull @Valid ConnectionConfi return updateConnectionConfig(id, connection, true); } + @SkipAuthorize("odc internal usage") public ConnectionConfig updateWithoutPermissionCheck(@NotNull Long id, @NotNull @Valid ConnectionConfig connection) { return updateConnectionConfig(id, connection, false); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index 564cfdc948..bd23675cd4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -548,6 +548,7 @@ public Boolean internalSyncDataSourceSchemas(@NonNull Long dataSourceId) throws } } + @SkipAuthorize("odc internal usage") public int updateEnvironmentIdByConnectionId(@NotNull Long environmentId, @NotNull Long connectionId) { return databaseRepository.setEnvironmentIdByConnectionId(environmentId, connectionId); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java index 0cf613ec4a..ce7af41500 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java @@ -209,6 +209,7 @@ private DBSchemaSyncer getSyncerByTableType(@NotNull DBObjectType tableType) { } // sync normal table + @SkipAuthorize("odc internal usage") public void syncDBTables(@NotNull Connection connection, @NotNull Database database, @NotNull DialectType dialectType) throws InterruptedException { syncDBTables(connection, database, dialectType, dbTableSyncer); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index 177af4215e..b500e4bf8b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -129,6 +129,7 @@ public List killSessionOrQuery(KillSessionOrQueryReq request) { } @Override + @SkipAuthorize("odc internal usage") public boolean supportKillConsoleQuery(ConnectionSession session) { if (Objects.nonNull(session) && ConnectionSessionUtil.isLogicalSession(session)) { return false; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java index b2be8729b1..949b34a5fe 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.metadb.iam.UserRoleRepository; @@ -41,6 +42,7 @@ public class GlobalResourceRoleService { @Autowired private UserRoleRepository userRoleRepository; + @SkipAuthorize("odc internal usage") public List findGlobalResourceRoleUsersByOrganizationId(Long organizationId) { return userRoleRepository.findByOrganizationIdAndNameIn( organizationId, @@ -48,6 +50,7 @@ public List findGlobalResourceRoleUsersByOrganizationId( GlobalResourceRoleUtil.GLOBAL_PROJECT_SECURITY_ADMINISTRATOR)); } + @SkipAuthorize("odc internal usage") public List findGlobalResourceRoleUsersByOrganizationIdAndUserId(Long organizationId, Long userId) { return userRoleRepository.findByOrganizationIdAndUserIdAndNameIn(organizationId, userId, @@ -55,7 +58,7 @@ public List findGlobalResourceRoleUsersByOrganizationIdA GlobalResourceRoleUtil.GLOBAL_PROJECT_SECURITY_ADMINISTRATOR)); } - + @SkipAuthorize("odc internal usage") public List findGlobalResourceRoleUsersByOrganizationIdAndRole(Long organizationId, ResourceType resourceType, ResourceRoleName resourceRoleName) { if (resourceType != ResourceType.ODC_PROJECT) { @@ -65,6 +68,7 @@ public List findGlobalResourceRoleUsersByOrganizationIdA organizationId, Arrays.asList(GlobalResourceRoleUtil.getGlobalRoleName(resourceRoleName))); } + @SkipAuthorize("odc internal usage") public List findGlobalResourceRoleUsersByOrganizationIdAndRoleIn(Long organizationId, Set resourceRoleNames) { if (CollectionUtils.isEmpty(resourceRoleNames)) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java index 5faca9a970..c16582ecd3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleService.java @@ -390,6 +390,7 @@ private UserResourceRole fromEntity(UserResourceRoleEntity entity, ResourceRoleE return model; } + @SkipAuthorize("internal usage") public static UserResourceRoleEntity toEntity(UserResourceRole model) { UserResourceRoleEntity entity = new UserResourceRoleEntity(); entity.setResourceId(model.getResourceId()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java index 74d2285eb3..5e8b77c7d5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java @@ -462,12 +462,14 @@ public BinaryDataResult getBatchImportTemplateFile() throws IOException { } } + @SkipAuthorize("odc internal usage") public Set getCurrentUserResourceRoleIdentifiers() { long currentUserId = authenticationFacade.currentUserId(); long currentOrganizationId = authenticationFacade.currentOrganizationId(); return resourceRoleService.getResourceRoleIdentifiersByUserId(currentOrganizationId, currentUserId); } + @SkipAuthorize("odc internal usage") public Set getCurrentUserJoinedProjectIds() { return resourceRoleService.getProjectId2ResourceRoleNames().keySet(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java index 64a77e37e9..80aa1026b8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java @@ -244,6 +244,7 @@ public IntegrationConfig update(@NotNull Long id, @NotNull @Valid IntegrationCon return new IntegrationConfig(entity); } + @SkipAuthorize("odc internal usage") public IntegrationConfig getDecodeConfig(IntegrationEntity entity) { IntegrationConfig integrationConfig = new IntegrationConfig(entity); String secret = decodeSecret(entity.getSecret(), entity.getSalt(), entity.getOrganizationId()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java index 46ec6609c8..0a7fb47d0d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java @@ -134,6 +134,7 @@ private static String resolveSamlRegistrationId(HttpServletRequest request) { return null; } + @SkipAuthorize public void saveSamlInfoIfNeed(String info) { HttpServletRequest currentRequest = WebRequestUtils.getCurrentRequest(); Verify.notNull(currentRequest, "currentRequest"); @@ -258,6 +259,7 @@ public void abortIfLdapTestLogin() { } } + @SkipAuthorize public void abortIfSamlTestLogin() { HttpServletRequest currentRequest = WebRequestUtils.getCurrentRequest(); if (currentRequest == null) { diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/DefaultSamlUserService.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/DefaultSamlUserService.java index 7c7bf0ac5b..7ae43048b8 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/DefaultSamlUserService.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/DefaultSamlUserService.java @@ -19,6 +19,7 @@ import org.springframework.security.saml2.provider.service.authentication.Saml2Authentication; import org.springframework.stereotype.Service; +import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.service.iam.auth.MappingRuleConvert; import com.oceanbase.odc.service.iam.auth.SsoUserDetailService; import com.oceanbase.odc.service.iam.auth.oauth2.MappingResult; @@ -33,6 +34,7 @@ public class DefaultSamlUserService { @Autowired private MappingRuleConvert mappingRuleConvert; + @SkipAuthorize public User loadUser(Saml2Authentication saml2Authentication) { MappingResult mappingResult = mappingRuleConvert.resolveSamlMappingResult(saml2Authentication); return ssoUserDetailService.getOrCreateUser(mappingResult); From 6b0e0c804b375574595d6c19018d86a7c38a8936 Mon Sep 17 00:00:00 2001 From: kiko Date: Mon, 13 Jan 2025 11:38:03 +0800 Subject: [PATCH 102/118] fix(schedule):delete without running task #4157 --- .../com/oceanbase/odc/service/schedule/ScheduleService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 4bc5fca2fd..e6503004bb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -326,9 +326,9 @@ public ChangeScheduleResp changeSchedule(ScheduleChangeParams req) { null, "Pause schedule is not allowed."); } if (req.getOperationType() == OperationType.DELETE) { - PreConditions.validRequestState(targetSchedule.getStatus() == ScheduleStatus.TERMINATED - || targetSchedule.getStatus() == ScheduleStatus.COMPLETED - || !hasExecutingTask(targetSchedule.getId()), ErrorCodes.DeleteNotAllowed, null, + PreConditions.validRequestState((targetSchedule.getStatus() == ScheduleStatus.TERMINATED + || targetSchedule.getStatus() == ScheduleStatus.COMPLETED) + && !hasExecutingTask(targetSchedule.getId()), ErrorCodes.DeleteNotAllowed, null, "Delete schedule is not allowed."); } } From 406e9b6dfa33b2b2ce08e12fac3553d1297413f2 Mon Sep 17 00:00:00 2001 From: MarkPotato777 Date: Mon, 13 Jan 2025 14:14:57 +0800 Subject: [PATCH 103/118] remove duplicated annotation --- .../oceanbase/odc/service/db/session/DefaultDBSessionManage.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index c11c329278..b500e4bf8b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -128,7 +128,6 @@ public List killSessionOrQuery(KillSessionOrQueryReq request) { } } - @SkipAuthorize @Override @SkipAuthorize("odc internal usage") public boolean supportKillConsoleQuery(ConnectionSession session) { From 49f2ab444dba91a66be0c5eb807d475d51c8dda2 Mon Sep 17 00:00:00 2001 From: zhangxiao <140503120+PeachThinking@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:40:37 +0800 Subject: [PATCH 104/118] fix(sql-parser): failed to parse table ddl when create fulltext key with parser #4154 * fix failed to parse table ddl when create fulltext index with parser * with parser adaption for native mysql * fix mysql g4 create fulltext index with parser * fix ut test --- .../schema/mysql/OBMySQLSchemaAccessor.java | 25 +++++++++------ .../tools/dbbrowser/parser/SqlParserTest.java | 2 +- .../mysql/MySQLIndexOptionsFactory.java | 2 +- .../main/resources/obmysql/sql/OBParser.g4 | 2 +- .../adapter/MySQLTableElementFactoryTest.java | 32 ++++++++++++++----- 5 files changed, 42 insertions(+), 21 deletions(-) diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java index a9b9f9a4e1..bc5720e892 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java @@ -289,16 +289,21 @@ private void parseDdlToSetIndexInfo(String ddl, List indexList) { fillWarning(indexList, DBObjectType.INDEX, "table ddl is blank, can not set index range by parse ddl"); return; } - ParseSqlResult result = SqlParser.parseMysql(ddl); - if (CollectionUtils.isEmpty(result.getIndexes())) { - fillWarning(indexList, DBObjectType.INDEX, "parse index DDL failed"); - } else { - indexList.forEach(index -> result.getIndexes().forEach(dbIndex -> { - if (StringUtils.equals(index.getName(), dbIndex.getName())) { - index.setGlobal("GLOBAL".equalsIgnoreCase(dbIndex.getRange().name())); - index.setColumnGroups(dbIndex.getColumnGroups()); - } - })); + try { + ParseSqlResult result = SqlParser.parseMysql(ddl); + if (CollectionUtils.isEmpty(result.getIndexes())) { + fillWarning(indexList, DBObjectType.INDEX, "parse index DDL failed"); + } else { + indexList.forEach(index -> result.getIndexes().forEach(dbIndex -> { + if (StringUtils.equals(index.getName(), dbIndex.getName())) { + index.setGlobal("GLOBAL".equalsIgnoreCase(dbIndex.getRange().name())); + index.setColumnGroups(dbIndex.getColumnGroups()); + } + })); + } + } catch (Exception e) { + fillWarning(indexList, DBObjectType.INDEX, "failed to set index info by parse ddl"); + log.warn("failed to set index info by parse ddl:{}", ddl, e); } } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java index 6e938d1898..07057790cc 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java @@ -224,7 +224,7 @@ public void test_parse_mysql_fulltext_index_range() { + " `title` varchar(200) DEFAULT NULL,\n" + " `content` text DEFAULT NULL,\n" + " PRIMARY KEY (`id`), \n" - + " FULLTEXT KEY `title` (`title`, `content`) CTXCAT(`title`, `content`) WITH PARSER 'TAOBAO_CHN' BLOCK_SIZE 16384\n" + + " FULLTEXT KEY `title` (`title`, `content`) CTXCAT(`title`, `content`) WITH PARSER TAOBAO_CHN BLOCK_SIZE 16384\n" + ") "; ParseSqlResult result = SqlParser.parseMysql(sql); Assert.assertEquals(1, result.getIndexes().size()); diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java index 2888e4a780..1ba44cf805 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java @@ -80,7 +80,7 @@ public IndexOptions visitOpt_index_options(Opt_index_optionsContext ctx) { } else if (option.CTXCAT() != null) { indexOptions.setCtxcat(getReference(option)); } else if (option.WITH() != null && option.PARSER() != null) { - indexOptions.setWithParser(option.STRING_VALUE().getText()); + indexOptions.setWithParser(option.relation_name().getText()); } else if (option.WITH() != null && option.ROWID() != null) { indexOptions.setWithRowId(true); } else if (option.WITH() != null && option.vec_index_params() != null) { diff --git a/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 b/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 index 472619c695..529f4bb9e0 100644 --- a/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 +++ b/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 @@ -1861,7 +1861,7 @@ index_option | COMMENT STRING_VALUE | (STORING|CTXCAT) LeftParen column_name_list RightParen | WITH ROWID - | WITH PARSER STRING_VALUE + | WITH PARSER relation_name | WITH LeftParen vec_index_params RightParen | index_using_algorithm | visibility_option diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java index aae65c06fd..6e928f7b81 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java @@ -693,7 +693,7 @@ public void generate_columnDefCheckConstraint1_generateSuccees() { @Test public void generate_indexBtree_succeed() { StatementFactory factory = new MySQLTableElementFactory( - getTableElementContext("index idx_name using btree (col, col1) global with parser 'aaaa'")); + getTableElementContext("index idx_name using btree (col, col1) global with parser `aaaa`")); OutOfLineIndex actual = (OutOfLineIndex) factory.generate(); SortColumn s1 = new SortColumn(new ColumnReference(null, null, "col")); @@ -702,7 +702,7 @@ public void generate_indexBtree_succeed() { IndexOptions indexOptions = new IndexOptions(); indexOptions.setUsingBtree(true); indexOptions.setGlobal(true); - indexOptions.setWithParser("'aaaa'"); + indexOptions.setWithParser("`aaaa`"); expect.setIndexOptions(indexOptions); Assert.assertEquals(expect, actual); } @@ -711,7 +711,7 @@ public void generate_indexBtree_succeed() { public void generate_exprIndexBtree_succeed() { StatementFactory factory = new MySQLTableElementFactory( getTableElementContext( - "index idx_name using btree ((CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 33 END)) global with parser 'aaaa'")); + "index idx_name using btree ((CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 33 END)) global with parser `aaaa`")); OutOfLineIndex actual = (OutOfLineIndex) factory.generate(); List whenClauses = new ArrayList<>(); @@ -725,7 +725,7 @@ public void generate_exprIndexBtree_succeed() { IndexOptions indexOptions = new IndexOptions(); indexOptions.setUsingBtree(true); indexOptions.setGlobal(true); - indexOptions.setWithParser("'aaaa'"); + indexOptions.setWithParser("`aaaa`"); expect.setIndexOptions(indexOptions); Assert.assertEquals(expect, actual); } @@ -911,7 +911,7 @@ public void generate_indexPrimaryKeyNoyAlgorithmAndComment_succeed() { public void generate_uniqueIndexColumnAscId_succeed() { StatementFactory factory = new MySQLTableElementFactory( getTableElementContext( - "unique index idx_name using btree (col asc id 16, col1) global with parser 'aaaa'")); + "unique index idx_name using btree (col asc id 16, col1) global with parser `aaaa`")); TableElement actual = factory.generate(); SortColumn s1 = new SortColumn(new ColumnReference(null, null, "col")); @@ -922,7 +922,7 @@ public void generate_uniqueIndexColumnAscId_succeed() { IndexOptions indexOptions = new IndexOptions(); indexOptions.setUsingBtree(true); indexOptions.setGlobal(true); - indexOptions.setWithParser("'aaaa'"); + indexOptions.setWithParser("`aaaa`"); state.setIndexOptions(indexOptions); OutOfLineConstraint expect = new OutOfLineConstraint(state, Arrays.asList(s1, s2)); expect.setUniqueKey(true); @@ -977,7 +977,7 @@ public void generate_uniqueIndexAutoPartition_succeed() { @Test public void generate_uniqueIndexColumnAscIdNoIndexOps_succeed() { StatementFactory factory = new MySQLTableElementFactory( - getTableElementContext("unique index idx_name (col asc id 16, col1) global with parser 'aaaa'")); + getTableElementContext("unique index idx_name (col asc id 16, col1) global with parser `aaaa`")); TableElement actual = factory.generate(); SortColumn s1 = new SortColumn(new ColumnReference(null, null, "col")); @@ -987,7 +987,7 @@ public void generate_uniqueIndexColumnAscIdNoIndexOps_succeed() { ConstraintState state = new ConstraintState(); IndexOptions indexOptions = new IndexOptions(); indexOptions.setGlobal(true); - indexOptions.setWithParser("'aaaa'"); + indexOptions.setWithParser("`aaaa`"); state.setIndexOptions(indexOptions); OutOfLineConstraint expect = new OutOfLineConstraint(state, Arrays.asList(s1, s2)); expect.setUniqueKey(true); @@ -1174,6 +1174,22 @@ public void generate_outOfLineIndexVectorIndex_succeed() { Assert.assertEquals(expected, actual); } + @Test + public void generate_outOfLineIndexFullTextIndex_getWithParser_Succeed() { + StatementFactory factory = new MySQLTableElementFactory( + getTableElementContext("FULLTEXT KEY idx1 (name) WITH PARSER space")); + TableElement actual = factory.generate(); + + OutOfLineIndex expected = new OutOfLineIndex("idx1", + Collections.singletonList(new SortColumn(new ColumnReference(null, null, "name")))); + expected.setFullText(true); + IndexOptions indexOptions = new IndexOptions(); + indexOptions.setWithParser("space"); + expected.setIndexOptions(indexOptions); + + Assert.assertEquals(expected, actual); + } + private Table_elementContext getTableElementContext(String str) { OBLexer lexer = new OBLexer(CharStreams.fromString(str)); CommonTokenStream tokens = new CommonTokenStream(lexer); From aab5e3bc9a0f93a14bde828ab1357db686fbe2a1 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Mon, 13 Jan 2025 14:59:20 +0800 Subject: [PATCH 105/118] security: add @SkipAuthorize for IntegrationService #4160 --- .../oceanbase/odc/service/integration/IntegrationService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java index 80aa1026b8..1cde2f007d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java @@ -380,6 +380,7 @@ private void updateCache(Long key) { } } + @SkipAuthorize("odc internal usage") public SSOCredential generateSSOCredential() { return new SSOCredential(samlCredentialManager.generateCertWithCachedPrivateKey()); } From 6f5fba07cdda992aa6d3aeaad8dd0591e38d3dbb Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Mon, 13 Jan 2025 15:18:11 +0800 Subject: [PATCH 106/118] build: upgrade db-browser to 1.2.1 #4159 --- libs/db-browser/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/db-browser/pom.xml b/libs/db-browser/pom.xml index a40d4157de..8abba2905d 100644 --- a/libs/db-browser/pom.xml +++ b/libs/db-browser/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.oceanbase db-browser - 1.2.0 + 1.2.1 db-browser https://github.com/oceanbase/odc/tree/main/libs/db-browser diff --git a/pom.xml b/pom.xml index aa9c591c75..103b9d8d50 100644 --- a/pom.xml +++ b/pom.xml @@ -93,7 +93,7 @@ 4.20.19.ALL 4.10.0 2.10.0 - 1.2.0 + 1.2.1 1.4.0 3.10.0 1.64 From 57667b396e753ec7c04c529e21ceb9407514932d Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Mon, 13 Jan 2025 19:59:46 +0800 Subject: [PATCH 107/118] build: update 4.3.3 submodule #4163 --- client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client b/client index 97074b7ddf..5483141527 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 97074b7ddffb212b0a43687bb88665d94450eb7b +Subproject commit 5483141527674c54261206470deddcdf96aaead7 From ebe1aafb7ecbf64744e56d570730e9d20a236102 Mon Sep 17 00:00:00 2001 From: guowl3 Date: Mon, 13 Jan 2025 20:27:43 +0800 Subject: [PATCH 108/118] feat(dlm): support filesystem (#4151) * supports create and test file system datasource * add i18 * fix access denied * fix access denied * fix access denied * fix delete obj failed * support sync database * set default schema * fix * support archive to file system * upgrade hadoop-common from 3.3.4 to 3.3.6 * bugfix * bugfix * bugfix * bugfix * pull mode * revert * cannot sync oss datasource * code format * code format * opt dlm task implements * fix npe * store partition range in disk * bugfix * code format * opt error message * opt task log * create table for all type * enable save point * opt log info * bugfix * bugfix * modify oss configuration * modify oss configuration * fix object key * use file url as schema name * opt oracle username * rollback commit * enable ob sync table structure * adapt check point * fix region is null * update status when task failed * bugfix * merge 4.3.x * record global range in statistic * record global range in statistic * upgrade sdk version to 1.2.0 * rsp comment * rsp comment * rsp comments --- pom.xml | 4 +- .../odc/core/shared/constant/ConnectType.java | 4 + .../odc/core/shared/constant/DialectType.java | 1 + .../odc/core/shared/constant/ErrorCodes.java | 9 +- .../resources/i18n/ErrorMessages.properties | 5 + .../i18n/ErrorMessages_zh_CN.properties | 6 + .../i18n/ErrorMessages_zh_TW.properties | 3 + server/odc-server/src/main/resources/data.sql | 8 +- .../src/main/resources/log4j2-task.xml | 13 ++ .../connection/ConnectionEventPublisher.java | 52 +++++ .../service/connection/ConnectionService.java | 11 +- .../service/connection/ConnectionTesting.java | 6 + .../connection/ConnectionValidator.java | 5 +- .../FileSystemConnectionTester.java | 148 ++++++++++++ .../connection/database/DatabaseService.java | 4 + .../event/UpsertDatasourceEvent.java | 38 ++++ .../listener/UpdateDatasourceListener.java | 88 +++++++ .../connection/model/ConnectionConfig.java | 7 + .../connection/model/TestConnectionReq.java | 19 ++ .../db/schema/DBSchemaSyncTaskManager.java | 4 + .../odc/service/dlm/DLMConfiguration.java | 14 +- .../odc/service/dlm/DLMJobFactory.java | 5 +- .../odc/service/dlm/DLMJobStore.java | 160 ++++--------- .../oceanbase/odc/service/dlm/DLMService.java | 3 + .../dlm/DLMTableStructureSynchronizer.java | 8 +- .../odc/service/dlm/DataArchiveJobStore.java | 214 ------------------ .../odc/service/dlm/DataSourceInfoMapper.java | 65 +++--- .../dlm/model/DataArchiveTableConfig.java | 10 + .../odc/service/dlm/model/DlmTableUnit.java | 9 + .../dlm/utils/TaskGeneratorMapper.java | 7 +- .../CreateFlowInstanceProcessAspect.java | 13 -- .../client/CloudObjectStorageClient.java | 6 +- .../cloud/CloudObjectStorage.java | 4 + .../cloud/client/AlibabaCloudClient.java | 17 ++ .../cloud/client/AmazonCloudClient.java | 17 ++ .../cloud/client/NullCloudClient.java | 7 + .../quartz/executor/AbstractQuartzJob.java | 13 ++ .../odc/service/schedule/ScheduleService.java | 12 +- .../service/schedule/job/AbstractDlmJob.java | 37 ++- .../schedule/job/DataArchiveDeleteJob.java | 1 + .../service/schedule/job/DataArchiveJob.java | 1 + .../schedule/job/DataArchiveRollbackJob.java | 1 + .../service/schedule/job/DataDeleteJob.java | 1 + .../schedule/model/DlmTableUnitStatistic.java | 11 + .../processor/AbstractDlmPreprocessor.java | 2 +- .../processor/DataArchivePreprocessor.java | 42 +--- .../base/dataarchive/DataArchiveTask.java | 203 ++++++++++------- .../processor/result/DLMResultProcessor.java | 2 +- .../odc/plugin/connect/api/TestResult.java | 16 ++ 49 files changed, 792 insertions(+), 544 deletions(-) create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionEventPublisher.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTester.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpsertDatasourceEvent.java create mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java delete mode 100644 server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataArchiveJobStore.java diff --git a/pom.xml b/pom.xml index 86af84cce5..92b55b325e 100644 --- a/pom.xml +++ b/pom.xml @@ -113,13 +113,13 @@ 1.2.19 1.5.4 - 3.3.4 + 3.3.6 4.1.94.Final 1.2.1 2.1.6 - 1.1.6.bp4 + 1.2.0 2.11.0 diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java index 5c336e4345..6d2572833b 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java @@ -35,6 +35,10 @@ public enum ConnectType { // reserved for future version ODP_SHARDING_OB_ORACLE(DialectType.OB_ORACLE), ORACLE(DialectType.ORACLE), + OSS(DialectType.FILE_SYSTEM), + OBS(DialectType.FILE_SYSTEM), + COS(DialectType.FILE_SYSTEM), + S3A(DialectType.FILE_SYSTEM), UNKNOWN(DialectType.UNKNOWN), ; diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java index 97b33431f8..b83c369e90 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java @@ -29,6 +29,7 @@ public enum DialectType { ODP_SHARDING_OB_MYSQL, DORIS, POSTGRESQL, + FILE_SYSTEM, UNKNOWN, ; diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java index ecd738b5b5..08940280af 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java @@ -170,6 +170,7 @@ public enum ErrorCodes implements ErrorCode { // Schedule AlterScheduleExists, InvalidCronExpression, + ScheduleIntervalTooShort, UpdateNotAllowed, PauseNotAllowed, DeleteNotAllowed, @@ -315,7 +316,13 @@ public enum ErrorCodes implements ErrorCode { * workspace */ WorkspaceDatabaseUserTypeMustBeAdmin, - ; + /** + * oss + */ + BucketNotExist, + InvalidAccessKeyId, + SignatureDoesNotMatch, + UnsupportedSyncTableStructure; @Override public String code() { diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties index b2479eebe8..3b4c0109b5 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties @@ -201,6 +201,11 @@ com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=Database access is denied beca com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=Query profile is only available for OceanBase Database with versions equal to or higher than {0}. com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=Someone has just modified this file. Please refresh the page to get the latest version and continue editing. com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=The database user type used in the workspace must be a super account. +com.oceanbase.odc.ErrorCodes.BucketNotExist=Bucket does not exist +com.oceanbase.odc.ErrorCodes.InvalidAccessKeyId=Invalid access key id +com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=Invalid access key secret +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=Sync table structure is not supported for {0} to {1} +com.oceanbase.odc.ErrorCodes.ScheduleIntervalTooShort=The execution interval is configured too short, please reconfigure. The minimum interval is: {0} seconds. com.oceanbase.odc.ErrorCodes.UpdateNotAllowed=Editing is not allowed in the current state. Please try again after disabling. com.oceanbase.odc.ErrorCodes.PauseNotAllowed=Disabling is not allowed in the current state, please check if there are any records in execution. com.oceanbase.odc.ErrorCodes.DeleteNotAllowed=Deletion is not allowed in the current state. diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties index e910200748..ec15c6c22e 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties @@ -204,3 +204,9 @@ com.oceanbase.odc.ErrorCodes.UpdateNotAllowed=当前状态下不允许编辑， com.oceanbase.odc.ErrorCodes.PauseNotAllowed=当前状态下不允许禁用，请检查是否存在执行中的记录 com.oceanbase.odc.ErrorCodes.DeleteNotAllowed=当前状态下不允许删除 + +com.oceanbase.odc.ErrorCodes.BucketNotExist=桶不存在 +com.oceanbase.odc.ErrorCodes.InvalidAccessKeyId=无效的 AccessKeyId +com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=无效的 AccessKeySecret +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=结构同步暂不支持 {0} 到 {1} +com.oceanbase.odc.ErrorCodes.ScheduleIntervalTooShort=执行间隔配置过短，请重新配置。最小间隔为：{0} 秒 \ No newline at end of file diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties index 6b05e76c53..d587dfe771 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties @@ -205,3 +205,6 @@ com.oceanbase.odc.ErrorCodes.PauseNotAllowed=當前狀態下不允許禁用， com.oceanbase.odc.ErrorCodes.DeleteNotAllowed=在目前狀態下不允許删除 + +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=結構同步暫不支持 {0} 到 {1} +com.oceanbase.odc.ErrorCodes.ScheduleIntervalTooShort=執行間隔設定過短，請重新設定。最小間隔應為：{0} 秒 \ No newline at end of file diff --git a/server/odc-server/src/main/resources/data.sql b/server/odc-server/src/main/resources/data.sql index 3e9ce4e440..b64d204ef5 100644 --- a/server/odc-server/src/main/resources/data.sql +++ b/server/odc-server/src/main/resources/data.sql @@ -859,4 +859,10 @@ INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('o INSERT INTO `config_system_configuration` (`key`, `value`, `application`, `profile`, `label`, `description`) VALUES ('odc.session.kill-query-or-session.max-supported-ob-version', '4.2.5', 'odc', 'default', 'master', 'Max OBVersion kill session or kill query supported, only take effect when value greater than 0') -ON DUPLICATE KEY UPDATE `id`=`id`; \ No newline at end of file +ON DUPLICATE KEY UPDATE `id`=`id`; +INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES('odc.task.dlm.session-limiting.enabled', 'true', +'Explosion-proof current limiting switch of mysql/oracle' ) +ON DUPLICATE KEY UPDATE `id` = `id`; +INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES('odc.task.dlm.session-limiting-ratio', '25', +'The ratio of oracle/mysql active sessions to the maximum number of connections allowed' ) +ON DUPLICATE KEY UPDATE `id` = `id`; \ No newline at end of file diff --git a/server/odc-server/src/main/resources/log4j2-task.xml b/server/odc-server/src/main/resources/log4j2-task.xml index d99b722e56..7c9e6965e1 100644 --- a/server/odc-server/src/main/resources/log4j2-task.xml +++ b/server/odc-server/src/main/resources/log4j2-task.xml @@ -64,6 +64,11 @@ + + + + + @@ -87,6 +92,11 @@ + + + + + @@ -117,6 +127,9 @@ + + + diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionEventPublisher.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionEventPublisher.java new file mode 100644 index 0000000000..4920c18554 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionEventPublisher.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.oceanbase.odc.common.event.AbstractEvent; +import com.oceanbase.odc.common.event.LocalEventPublisher; +import com.oceanbase.odc.service.connection.listener.UpdateDatasourceListener; + +import lombok.NonNull; + +/** + * @Author：tinker + * @Date: 2024/12/30 10:57 + * @Descripition: + */ + +@Component +public class ConnectionEventPublisher { + @Autowired + private LocalEventPublisher localEventPublisher; + + @Autowired + private UpdateDatasourceListener updateDatasourceListener; + + @PostConstruct + public void init() { + localEventPublisher.addEventListener(updateDatasourceListener); + } + + public void publishEvent(@NonNull T event) { + localEventPublisher.publishEvent(event); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java index 4900d717c5..6f3137542b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java @@ -105,6 +105,7 @@ import com.oceanbase.odc.service.connection.ConnectionStatusManager.CheckState; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.DatabaseSyncManager; +import com.oceanbase.odc.service.connection.event.UpsertDatasourceEvent; import com.oceanbase.odc.service.connection.model.ConnectProperties; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.connection.model.OBTenantEndpoint; @@ -221,6 +222,9 @@ public class ConnectionService { @Autowired private TransactionTemplate txTemplate; + @Autowired + private ConnectionEventPublisher connectionEventPublisher; + private final ConnectionMapper mapper = ConnectionMapper.INSTANCE; public static final String DEFAULT_MIN_PRIVILEGE = "read"; @@ -249,6 +253,7 @@ public ConnectionConfig create(@NotNull @Valid ConnectionConfig connection, @Not } }); databaseSyncManager.submitSyncDataSourceAndDBSchemaTask(saved); + connectionEventPublisher.publishEvent(new UpsertDatasourceEvent(saved)); return saved; } @@ -393,8 +398,11 @@ public ConnectionConfig getWithoutPermissionCheck(@NotNull Long id) { @SkipAuthorize("odc internal usage") public List listByOrganizationId(@NonNull Long organizationId) { - return entitiesToModels(repository.findByOrganizationIdOrderByNameAsc(organizationId), organizationId, true, + List connectionConfigs = entitiesToModels( + repository.findByOrganizationIdOrderByNameAsc(organizationId), organizationId, true, true); + fullFillAttributes(connectionConfigs); + return connectionConfigs; } @SkipAuthorize("odc internal usage") @@ -685,6 +693,7 @@ private ConnectionConfig updateConnectionConfig(Long id, ConnectionConfig connec } }); databaseSyncManager.submitSyncDataSourceAndDBSchemaTask(config); + connectionEventPublisher.publishEvent(new UpsertDatasourceEvent(config)); return config; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java index 666772cf20..ea7a4b31d5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java @@ -74,6 +74,8 @@ public class ConnectionTesting { private ConnectionSSLAdaptor connectionSSLAdaptor; @Autowired private CloudMetadataClient cloudMetadataClient; + @Autowired + private FileSystemConnectionTester fileSystemConnectionTesting; @Value("${odc.sdk.test-connect.query-timeout-seconds:2}") private int queryTimeoutSeconds = 2; @@ -102,6 +104,9 @@ public ConnectionTestResult test(@NotNull @Valid TestConnectionReq req) { public ConnectionTestResult test(@NonNull ConnectionConfig config) { ConnectType type = config.getType(); + if (type.getDialectType() == DialectType.FILE_SYSTEM) { + return fileSystemConnectionTesting.test(config); + } try { /** * 进行连接测试时需要关注的值有一个 {@link ConnectType}， 容易产生问题信息主要是两个：{@code username}, {@code defaultSchema} 首先分析 @@ -232,6 +237,7 @@ private ConnectionConfig reqToConnectionConfig(TestConnectionReq req) { config.setServiceName(req.getServiceName()); config.setUserRole(req.getUserRole()); config.setCatalogName(req.getCatalogName()); + config.setRegion(req.getRegion()); OBTenantEndpoint endpoint = req.getEndpoint(); if (Objects.nonNull(endpoint) && OceanBaseAccessMode.IC_PROXY == endpoint.getAccessMode()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java index b2fd42c5a5..fcab8f4504 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.AccessDeniedException; import com.oceanbase.odc.service.collaboration.environment.EnvironmentService; @@ -43,7 +44,9 @@ public class ConnectionValidator { void validateForUpsert(ConnectionConfig connection) { PreConditions.notNull(connection, "connection"); PreConditions.notBlank(connection.getHost(), "connection.host"); - PreConditions.notNull(connection.getPort(), "connection.port"); + if (connection.getDialectType() != DialectType.FILE_SYSTEM) { + PreConditions.notNull(connection.getPort(), "connection.port"); + } PreConditions.validNotSqlInjection(connection.getUsername(), "username"); PreConditions.validNotSqlInjection(connection.getClusterName(), "clusterName"); PreConditions.validNotSqlInjection(connection.getTenantName(), "tenantName"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTester.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTester.java new file mode 100644 index 0000000000..43d5c37be3 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTester.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; +import java.util.Collections; +import java.util.UUID; + +import org.springframework.stereotype.Component; + +import com.aliyun.oss.OSSErrorCode; +import com.aliyun.oss.OSSException; +import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.plugin.connect.api.TestResult; +import com.oceanbase.odc.service.cloud.model.CloudProvider; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.connection.model.ConnectionTestResult; +import com.oceanbase.odc.service.objectstorage.cloud.CloudResourceConfigurations; +import com.oceanbase.odc.service.objectstorage.cloud.client.CloudClient; +import com.oceanbase.odc.service.objectstorage.cloud.client.CloudException; +import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsRequest; +import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectMetadata; +import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; +import com.oceanbase.tools.migrator.common.exception.UnExpectedException; + +import lombok.NonNull; + +/** + * @Author：tinker + * @Date: 2024/11/19 11:24 + * @Descripition: + */ + +@Component +public class FileSystemConnectionTester { + + private static final String COS_ENDPOINT_PATTERN = "cos.{0}.myqcloud.com"; + private static final String OBS_ENDPOINT_PATTERN = "obs.{0}.myhuaweicloud.com"; + private static final String OSS_ENDPOINT_PATTERN = "oss-{0}.aliyuncs.com"; + private static final String S3_ENDPOINT_GLOBAL_PATTERN = "s3.{0}.amazonaws.com"; + private static final String S3_ENDPOINT_CN_PATTERN = "s3.{0}.amazonaws.com.cn"; + + private static final String TMP_FILE_NAME_PREFIX = "odc-test-object-"; + private static final String TMP_TEST_DATA = "This is a test object to check read and write permissions."; + + public ConnectionTestResult test(@NonNull ConnectionConfig config) { + PreConditions.notBlank(config.getPassword(), "AccessKeySecret"); + PreConditions.notBlank(config.getRegion(), "Region"); + URI uri = URI.create(config.getHost()); + ObjectStorageConfiguration storageConfig = new ObjectStorageConfiguration(); + storageConfig.setAccessKeyId(config.getUsername()); + storageConfig.setAccessKeySecret(config.getPassword()); + storageConfig.setBucketName(uri.getAuthority()); + storageConfig.setRegion(config.getRegion()); + storageConfig.setCloudProvider(getCloudProvider(config.getType())); + storageConfig.setPublicEndpoint(getEndPointByRegion(config.getType(), config.getRegion())); + try { + CloudClient cloudClient = + new CloudResourceConfigurations.CloudClientBuilder().generateCloudClient(storageConfig); + String objectKey = uri.getPath().endsWith("/") ? uri.getAuthority() + uri.getPath() + generateTempFileName() + : uri.getAuthority() + uri.getPath() + "/" + generateTempFileName(); + cloudClient.putObject(storageConfig.getBucketName(), objectKey, + new ByteArrayInputStream(TMP_TEST_DATA.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); + DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(); + deleteObjectsRequest.setBucketName(storageConfig.getBucketName()); + deleteObjectsRequest.setKeys(Collections.singletonList(objectKey)); + cloudClient.deleteObjects(deleteObjectsRequest); + return ConnectionTestResult.success(config.getType()); + } catch (CloudException e) { + if (e.getCause() != null && e.getCause() instanceof OSSException) { + OSSException cause = (OSSException) e.getCause(); + switch (cause.getErrorCode()) { + case OSSErrorCode.ACCESS_DENIED: + return new ConnectionTestResult(TestResult.akAccessDenied(storageConfig.getAccessKeyId()), + config.getType()); + case OSSErrorCode.INVALID_ACCESS_KEY_ID: + return new ConnectionTestResult(TestResult.invalidAccessKeyId(storageConfig.getAccessKeyId()), + config.getType()); + case OSSErrorCode.SIGNATURE_DOES_NOT_MATCH: + return new ConnectionTestResult( + TestResult.signatureDoesNotMatch(storageConfig.getAccessKeyId()), config.getType()); + case OSSErrorCode.NO_SUCH_BUCKET: + return new ConnectionTestResult(TestResult.bucketNotExist(storageConfig.getBucketName()), + config.getType()); + default: + return new ConnectionTestResult(TestResult.unknownError(e), config.getType()); + } + } + // TODO:process s3 error message + return new ConnectionTestResult(TestResult.unknownError(e), config.getType()); + } + } + + private CloudProvider getCloudProvider(ConnectType type) { + switch (type) { + case COS: + return CloudProvider.TENCENT_CLOUD; + case OBS: + return CloudProvider.HUAWEI_CLOUD; + case S3A: + return CloudProvider.AWS; + case OSS: + return CloudProvider.ALIBABA_CLOUD; + default: + throw new UnExpectedException(); + } + } + + private static String getEndPointByRegion(ConnectType type, String region) { + switch (type) { + case COS: + return MessageFormat.format(COS_ENDPOINT_PATTERN, region); + case OSS: + return MessageFormat.format(OSS_ENDPOINT_PATTERN, region); + case OBS: + return MessageFormat.format(OBS_ENDPOINT_PATTERN, region); + case S3A: + // Note there is a difference of Top-Level Domain between cn and global regions. + if (region.startsWith("cn-")) { + return MessageFormat.format(S3_ENDPOINT_CN_PATTERN, region); + } + return MessageFormat.format(S3_ENDPOINT_GLOBAL_PATTERN, region); + default: + throw new IllegalArgumentException("regionToEndpoint is not applicable for storageType " + type); + } + } + + private String generateTempFileName() { + return TMP_FILE_NAME_PREFIX + UUID.randomUUID(); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index bd23675cd4..9c27b53853 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -65,6 +65,7 @@ import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.OrganizationType; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; @@ -528,6 +529,9 @@ public Boolean internalSyncDataSourceSchemas(@NonNull Long dataSourceId) throws Optional organizationOpt = Optional.empty(); try { connection = connectionService.getForConnectionSkipPermissionCheck(dataSourceId); + if (connection.getDialectType() == DialectType.FILE_SYSTEM) { + return true; + } horizontalDataPermissionValidator.checkCurrentOrganization(connection); organizationOpt = organizationService.get(connection.getOrganizationId()); Organization organization = diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpsertDatasourceEvent.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpsertDatasourceEvent.java new file mode 100644 index 0000000000..cb4f9bcd28 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpsertDatasourceEvent.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.event; + +import com.oceanbase.odc.common.event.AbstractEvent; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; + +import lombok.Getter; + +/** + * @Author：tianke + * @Date: 2024/12/30 10:41 + * @Descripition: + */ +public class UpsertDatasourceEvent extends AbstractEvent { + + @Getter + private ConnectionConfig connectionConfig; + + public UpsertDatasourceEvent(ConnectionConfig connectionConfig) { + super(connectionConfig, "UpsertDatasourceEvent"); + this.connectionConfig = connectionConfig; + + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java new file mode 100644 index 0000000000..f52b9832bb --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.listener; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import com.oceanbase.odc.common.event.AbstractEventListener; +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.metadb.connection.DatabaseEntity; +import com.oceanbase.odc.metadb.connection.DatabaseRepository; +import com.oceanbase.odc.service.connection.database.model.DatabaseSyncStatus; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; +import com.oceanbase.odc.service.connection.event.UpsertDatasourceEvent; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.db.schema.model.DBObjectSyncStatus; + +import lombok.extern.slf4j.Slf4j; + +/** + * @Author：tinker + * @Date: 2024/12/30 10:47 + * @Descripition: + */ +@Slf4j +@Component +public class UpdateDatasourceListener extends AbstractEventListener { + + @Autowired + private DatabaseRepository databaseRepository; + + @Override + public void onEvent(UpsertDatasourceEvent event) { + + ConnectionConfig connectionConfig = event.getConnectionConfig(); + if (connectionConfig.getDialectType() != DialectType.FILE_SYSTEM) { + return; + } + List byConnectionId = databaseRepository.findByConnectionId(connectionConfig.getId()); + DatabaseEntity entity = null; + if (!CollectionUtils.isEmpty(byConnectionId)) { + List toBeDelete = byConnectionId.stream().filter( + o -> !connectionConfig.getHost().equals(o.getName())).map(DatabaseEntity::getId).collect( + Collectors.toList()); + if (!toBeDelete.isEmpty()) { + databaseRepository.deleteAllById(toBeDelete); + } + Optional existed = byConnectionId.stream().filter( + o -> connectionConfig.getHost().equals(o.getName())).findFirst(); + if (existed.isPresent()) { + entity = existed.get(); + } + } + // create or update + entity = entity == null ? new DatabaseEntity() : entity; + entity.setDatabaseId(com.oceanbase.odc.common.util.StringUtils.uuid()); + entity.setOrganizationId(connectionConfig.getOrganizationId()); + entity.setName(connectionConfig.getHost()); + entity.setProjectId(connectionConfig.getProjectId()); + entity.setConnectionId(connectionConfig.getId()); + entity.setEnvironmentId(connectionConfig.getEnvironmentId()); + entity.setSyncStatus(DatabaseSyncStatus.SUCCEEDED); + entity.setExisted(true); + entity.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); + entity.setConnectType(connectionConfig.getType()); + entity.setType(DatabaseType.PHYSICAL); + databaseRepository.save(entity); + + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java index b9bd8d1778..679ba0fdad 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java @@ -546,6 +546,13 @@ public String getCloudProvider() { return o == null ? null : o.toString(); } + public void setRegion(@NotNull String region) { + if (this.attributes == null) { + attributes = new HashMap<>(); + } + attributes.put(REGION, region); + } + public String getRegion() { if (this.attributes == null) { return null; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java index 8ca19b8fc9..1074d37549 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.connection.model; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -46,6 +47,9 @@ @Data @ToString(exclude = {"password"}) public class TestConnectionReq implements CloudConnectionConfig, SSLConnectionConfig { + + private static final String REGION = "region"; + /** * Connection ID，用于编辑连接页面未传密码参数时从已保存的连接信息中获取对应密码字段 */ @@ -157,6 +161,21 @@ public DialectType getDialectType() { return this.dialectType; } + public void setRegion(String region) { + if (this.attributes == null) { + attributes = new HashMap<>(); + } + attributes.put(REGION, region); + } + + public String getRegion() { + if (this.attributes == null) { + return null; + } + Object o = attributes.get(REGION); + return o == null ? null : o.toString(); + } + public static TestConnectionReq fromConnection(ConnectionConfig connection, ConnectionAccountType accountType) { PreConditions.notNull(accountType, "AccountType"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java index c4d443a1d1..c79e860b05 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java @@ -35,6 +35,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.ConflictException; import com.oceanbase.odc.core.shared.exception.NotFoundException; @@ -118,6 +119,9 @@ public void submitTaskByDatabases(@NonNull Collection databases) { } public void submitTaskByDataSource(@NonNull ConnectionConfig dataSource) { + if (dataSource.getDialectType() == DialectType.FILE_SYSTEM) { + return; + } List databases = databaseService.listExistDatabasesByConnectionId(dataSource.getId()); databases.removeIf(e -> (syncProperties.isBlockExclusionsWhenSyncDbSchemas() && syncProperties.getExcludeSchemas(dataSource.getDialectType()).contains(e.getName()) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java index 475793c5e3..c79c3dd3fe 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java @@ -16,11 +16,9 @@ package com.oceanbase.odc.service.dlm; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.oceanbase.tools.migrator.common.enums.ShardingStrategy; -import com.oceanbase.tools.migrator.core.IJobStore; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -36,9 +34,6 @@ @Configuration public class DLMConfiguration { - @Value("${odc.task.dlm.thread-pool-size:15}") - private int dlmThreadPoolSize; - @Value("${odc.task.dlm.single-task-read-write-ratio:0.5}") private double readWriteRatio; @@ -54,9 +49,10 @@ public class DLMConfiguration { @Value("${odc.task.dlm.default-scan-batch-size:10000}") private int defaultScanBatchSize; - @Bean - public DLMJobFactory dlmJobFactory(IJobStore jobStore) { - return new DLMJobFactory(jobStore); - } + @Value("${odc.task.dlm.session-limiting.enabled:true}") + private boolean sessionLimitingEnabled; + + @Value("${odc.task.dlm.session-limiting-ratio:25}") + private int sessionLimitingRatio; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java index e11e83d500..2829ddb2ab 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java @@ -18,9 +18,9 @@ import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.odc.service.dlm.model.DlmTableUnitParameters; import com.oceanbase.tools.migrator.common.dto.HistoryJob; -import com.oceanbase.tools.migrator.core.IJobStore; import com.oceanbase.tools.migrator.core.JobFactory; import com.oceanbase.tools.migrator.core.JobReq; +import com.oceanbase.tools.migrator.core.store.IJobStore; import com.oceanbase.tools.migrator.job.Job; import lombok.extern.slf4j.Slf4j; @@ -41,7 +41,6 @@ public Job createJob(DlmTableUnit parameters) { HistoryJob historyJob = new HistoryJob(); historyJob.setId(parameters.getDlmTableUnitId()); historyJob.setJobType(parameters.getType()); - historyJob.setTableId(-1L); historyJob.setPrintSqlTrace(false); historyJob.setSourceTable(parameters.getTableName()); historyJob.setTargetTable(parameters.getTargetTableName()); @@ -56,6 +55,8 @@ public Job createJob(DlmTableUnit parameters) { req.setHistoryJob(historyJob); req.setSourceDs(parameters.getSourceDatasourceInfo()); req.setTargetDs(parameters.getTargetDatasourceInfo()); + req.setSourceLimitConfig(parameters.getSourceLimitConfig()); + req.setTargetLimitConfig(parameters.getTargetLimitConfig()); return super.createJob(req); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index ba5d1a4be9..3d31cf1c77 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -16,37 +16,24 @@ package com.oceanbase.odc.service.dlm; import java.sql.Connection; -import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.LinkedList; import java.util.List; -import java.util.Map; import com.alibaba.druid.pool.DruidDataSource; -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.dlm.model.DlmTableUnit; -import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; -import com.oceanbase.odc.service.schedule.job.DLMJobReq; -import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; +import com.oceanbase.odc.service.session.factory.DruidDataSourceFactory; import com.oceanbase.tools.migrator.common.dto.JobStatistic; -import com.oceanbase.tools.migrator.common.dto.TableSizeInfo; import com.oceanbase.tools.migrator.common.dto.TaskGenerator; import com.oceanbase.tools.migrator.common.element.PrimaryKey; -import com.oceanbase.tools.migrator.common.exception.JobException; -import com.oceanbase.tools.migrator.common.exception.JobSqlException; -import com.oceanbase.tools.migrator.common.meta.TableMeta; -import com.oceanbase.tools.migrator.core.IJobStore; import com.oceanbase.tools.migrator.core.handler.genarator.GeneratorStatus; -import com.oceanbase.tools.migrator.core.handler.genarator.GeneratorType; -import com.oceanbase.tools.migrator.core.meta.ClusterMeta; -import com.oceanbase.tools.migrator.core.meta.JobMeta; import com.oceanbase.tools.migrator.core.meta.TaskMeta; -import com.oceanbase.tools.migrator.core.meta.TenantMeta; +import com.oceanbase.tools.migrator.core.store.IJobStore; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; /** @@ -58,18 +45,21 @@ public class DLMJobStore implements IJobStore { private DruidDataSource dataSource; - private boolean enableBreakpointRecovery = false; - private Map dlmTableUnits; - private Map jobParameters; + private boolean enableBreakpointRecovery = true; + @Setter + private DlmTableUnit dlmTableUnit; public DLMJobStore(ConnectionConfig metaDBConfig) { + try { + DruidDataSourceFactory druidDataSourceFactory = new DruidDataSourceFactory(metaDBConfig); + dataSource = (DruidDataSource) druidDataSourceFactory.getDataSource(); + } catch (Exception e) { + log.warn("Failed to connect to the meta database; closing save point."); + enableBreakpointRecovery = false; + } } - public void setDlmTableUnits(Map dlmTableUnits) { - this.dlmTableUnits = dlmTableUnits; - } - public void destroy() { if (dataSource == null) { return; @@ -82,7 +72,7 @@ public void destroy() { } @Override - public TaskGenerator getTaskGenerator(String generatorId, String jobId) throws SQLException { + public TaskGenerator getTaskGenerator(String jobId) throws SQLException { if (enableBreakpointRecovery) { try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( @@ -92,15 +82,14 @@ public TaskGenerator getTaskGenerator(String generatorId, String jobId) throws S if (resultSet.next()) { TaskGenerator taskGenerator = new TaskGenerator(); taskGenerator.setId(resultSet.getString("generator_id")); - taskGenerator.setGeneratorType(GeneratorType.valueOf(resultSet.getString("type"))); taskGenerator.setGeneratorStatus(GeneratorStatus.valueOf(resultSet.getString("status"))); taskGenerator.setJobId(jobId); taskGenerator.setTaskCount(resultSet.getInt("task_count")); taskGenerator - .setGeneratorSavePoint(PrimaryKey.valuesOf(resultSet.getString("primary_key_save_point"))); + .setPrimaryKeySavePoint(PrimaryKey.valuesOf(resultSet.getString("primary_key_save_point"))); taskGenerator.setProcessedDataSize(resultSet.getLong("processed_row_count")); taskGenerator.setProcessedDataSize(resultSet.getLong("processed_data_size")); - taskGenerator.setGeneratorPartitionSavepoint(resultSet.getString("partition_save_point")); + taskGenerator.setPartitionSavePoint(resultSet.getString("partition_save_point")); log.info("Load task generator success.jobId={}", jobId); return taskGenerator; } @@ -112,6 +101,16 @@ public TaskGenerator getTaskGenerator(String generatorId, String jobId) throws S @Override public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException { + taskGenerator.getPartName2MaxKey() + .forEach((k, v) -> dlmTableUnit.getStatistic().getPartName2MaxKey().put(k, v.getSqlString())); + taskGenerator.getPartName2MinKey() + .forEach((k, v) -> dlmTableUnit.getStatistic().getPartName2MinKey().put(k, v.getSqlString())); + if (taskGenerator.getGlobalMaxKey() != null) { + dlmTableUnit.getStatistic().setGlobalMaxKey(taskGenerator.getGlobalMaxKey().getSqlString()); + } + if (taskGenerator.getGlobalMinKey() != null) { + dlmTableUnit.getStatistic().setGlobalMinKey(taskGenerator.getGlobalMinKey().getSqlString()); + } if (enableBreakpointRecovery) { StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO dlm_task_generator "); @@ -130,11 +129,11 @@ public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException ps.setLong(3, taskGenerator.getProcessedDataSize()); ps.setLong(4, taskGenerator.getProcessedRowCount()); ps.setString(5, taskGenerator.getGeneratorStatus().name()); - ps.setString(6, GeneratorType.AUTO.name()); + ps.setString(6, ""); ps.setLong(7, taskGenerator.getTaskCount()); - ps.setString(8, taskGenerator.getGeneratorSavePoint() == null ? "" - : taskGenerator.getGeneratorSavePoint().toSqlString()); - ps.setString(9, taskGenerator.getGeneratorPartitionSavepoint()); + ps.setString(8, taskGenerator.getPrimaryKeySavePoint() == null ? "" + : taskGenerator.getPrimaryKeySavePoint().toSqlString()); + ps.setString(9, taskGenerator.getPartitionSavePoint()); if (ps.executeUpdate() == 1) { log.info("Update task generator success.jobId={}", taskGenerator.getJobId()); } else { @@ -145,39 +144,34 @@ public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException } @Override - public void bindGeneratorToJob(String s, TaskGenerator taskGenerator) throws SQLException { - - } - - @Override - public JobStatistic getJobStatistic(String s) throws JobException { + public JobStatistic getJobStatistic(String s) throws SQLException { return new JobStatistic(); } @Override - public void storeJobStatistic(JobMeta jobMeta) throws JobSqlException { - dlmTableUnits.get(jobMeta.getJobId()).getStatistic().setProcessedRowCount(jobMeta.getJobStat().getRowCount()); - dlmTableUnits.get(jobMeta.getJobId()).getStatistic() - .setProcessedRowsPerSecond(jobMeta.getJobStat().getAvgRowCount()); + public void storeJobStatistic(JobStatistic jobStatistic) throws SQLException { + dlmTableUnit.getStatistic() + .setProcessedRowCount(jobStatistic.getRowCount().get()); + dlmTableUnit.getStatistic() + .setProcessedRowsPerSecond(jobStatistic.getRowCountPerSeconds()); - dlmTableUnits.get(jobMeta.getJobId()).getStatistic().setReadRowCount(jobMeta.getJobStat().getReadRowCount()); - dlmTableUnits.get(jobMeta.getJobId()).getStatistic() - .setReadRowsPerSecond(jobMeta.getJobStat().getAvgReadRowCount()); + dlmTableUnit.getStatistic().setReadRowCount(jobStatistic.getReadRowCount().get()); + dlmTableUnit.getStatistic() + .setReadRowsPerSecond(jobStatistic.getReadRowCountPerSeconds()); } @Override - public List getTaskMeta(JobMeta jobMeta) throws SQLException { + public List loadUnfinishedTask(String generatorId) throws SQLException { if (enableBreakpointRecovery) { try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( "select * from dlm_task_unit where generator_id = ? AND status !='SUCCESS'")) { - ps.setString(1, jobMeta.getGenerator().getId()); + ps.setString(1, generatorId); ResultSet resultSet = ps.executeQuery(); List taskMetas = new LinkedList<>(); while (resultSet.next()) { TaskMeta taskMeta = new TaskMeta(); taskMeta.setTaskIndex(resultSet.getLong("task_index")); - taskMeta.setJobMeta(jobMeta); taskMeta.setGeneratorId(resultSet.getString("generator_id")); taskMeta.setTaskStatus(com.oceanbase.tools.migrator.common.enums.TaskStatus .valueOf(resultSet.getString("status"))); @@ -197,7 +191,6 @@ public List getTaskMeta(JobMeta jobMeta) throws SQLException { @Override public void storeTaskMeta(TaskMeta taskMeta) throws SQLException { if (enableBreakpointRecovery) { - log.info("start to store taskMeta:{}", taskMeta); StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO dlm_task_unit "); sb.append( @@ -228,7 +221,8 @@ public void storeTaskMeta(TaskMeta taskMeta) throws SQLException { } @Override - public Long getAbnormalTaskIndex(String jobId) { + public long getAbnormalTaskCount(String jobId) { + long count = 0; if (enableBreakpointRecovery) { try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( @@ -236,78 +230,14 @@ public Long getAbnormalTaskIndex(String jobId) { ps.setString(1, jobId); ResultSet resultSet = ps.executeQuery(); if (resultSet.next()) { - long count = resultSet.getLong(1); - return count > 0 ? count : null; + count = resultSet.getLong(1); } } catch (Exception ignored) { log.warn("Get abnormal task failed.jobId={}", jobId); } } - return null; + return count; } - @Override - public void updateTableSizeInfo(TableSizeInfo tableSizeInfo, long l) { - - } - @Override - public void updateLimiter(JobMeta jobMeta) { - try { - RateLimitConfiguration params; - if (jobParameters.containsKey(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG)) { - params = JsonUtils.fromJson( - jobParameters.get(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG), - RateLimitConfiguration.class); - } else { - DLMJobReq dlmJobReq = JsonUtils.fromJson( - jobParameters.get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), - DLMJobReq.class); - params = dlmJobReq.getRateLimit(); - } - if (params.getDataSizeLimit() != null) { - setClusterLimitConfig(jobMeta.getSourceCluster(), params.getDataSizeLimit()); - setClusterLimitConfig(jobMeta.getTargetCluster(), params.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getSourceTenant(), params.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getTargetTenant(), params.getDataSizeLimit()); - log.info("Update rate limit success,dataSizeLimit={}", params.getDataSizeLimit()); - } - if (params.getRowLimit() != null) { - setTableLimitConfig(jobMeta.getTargetTableMeta(), params.getRowLimit()); - setTableLimitConfig(jobMeta.getSourceTableMeta(), params.getRowLimit()); - log.info("Update rate limit success,rowLimit={}", params.getRowLimit()); - } - } catch (Exception e) { - log.warn("Update rate limit failed,errorMsg={}", e.getMessage()); - setClusterLimitConfig(jobMeta.getSourceCluster(), 1024); - setClusterLimitConfig(jobMeta.getTargetCluster(), 1024); - setTenantLimitConfig(jobMeta.getSourceTenant(), 1024); - setTenantLimitConfig(jobMeta.getTargetTenant(), 1024); - setTableLimitConfig(jobMeta.getTargetTableMeta(), 1000); - setTableLimitConfig(jobMeta.getSourceTableMeta(), 1000); - } - } - - public void setJobParameters(Map jobParameters) { - this.jobParameters = jobParameters; - } - - private void setClusterLimitConfig(ClusterMeta clusterMeta, long dataSizeLimit) { - clusterMeta.setReadSizeLimit(dataSizeLimit); - clusterMeta.setWriteSizeLimit(dataSizeLimit); - clusterMeta.setWriteUsedQuota(1); - clusterMeta.setReadUsedQuota(1); - } - - private void setTenantLimitConfig(TenantMeta tenantMeta, long dataSizeLimit) { - tenantMeta.setReadSizeLimit(dataSizeLimit); - tenantMeta.setWriteSizeLimit(dataSizeLimit); - tenantMeta.setWriteUsedQuota(1); - tenantMeta.setReadUsedQuota(1); - } - - private void setTableLimitConfig(TableMeta tableMeta, int rowLimit) { - tableMeta.setReadRowCountLimit(rowLimit); - tableMeta.setWriteRowCountLimit(rowLimit); - } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java index fe02913217..9862737d42 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java @@ -97,6 +97,9 @@ public void createOrUpdateDlmTableUnits(List dlmTableUnits) { DlmTableUnitEntity entity; if (entityOptional.isPresent()) { entity = entityOptional.get(); + if (entity.getStatus() == TaskStatus.DONE) { + return; + } entity.setStatistic(JsonUtils.toJson(o.getStatistic())); entity.setStatus(o.getStatus()); entity.setStartTime(o.getStartTime()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java index eee02f6a3d..c772f46ac9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java @@ -94,11 +94,12 @@ public static void sync(ConnectionConfig srcConfig, ConnectionConfig tgtConfig, DBTableStructureComparator comparator = new DBTableStructureComparator(tgtTableEditor, tgtConfig.getType().getDialectType(), srcConfig.getDefaultSchema(), tgtConfig.getDefaultSchema()); List changeSqlScript = new LinkedList<>(); + targetType.remove(DBObjectType.TABLE); if (tgtTable == null) { srcTable.setSchemaName(tgtConfig.getDefaultSchema()); srcTable.setName(tgtTableName); changeSqlScript.add(tgtTableEditor.generateCreateObjectDDL(srcTable)); - } else { + } else if (!targetType.isEmpty()) { DBObjectComparisonResult result = comparator.compare(srcTable, tgtTable); if (result.getComparisonResult() == ComparisonResult.INCONSISTENT) { changeSqlScript = result.getSubDBObjectComparisonResult().stream() @@ -128,11 +129,10 @@ public static void sync(ConnectionConfig srcConfig, ConnectionConfig tgtConfig, public static boolean isSupportedSyncTableStructure(DialectType srcType, String srcVersion, DialectType tgtType, String tgtVersion) { - // only supports MySQL or OBMySQL - if (!srcType.isMysql() || !tgtType.isMysql()) { + if (srcType != tgtType) { return false; } - if (srcType != tgtType) { + if (!srcType.isOceanbase() && !srcType.isMysql()) { return false; } // unsupported MySQL versions below 5.7.0 diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataArchiveJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataArchiveJobStore.java deleted file mode 100644 index f373059cb0..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataArchiveJobStore.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.dlm; - -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.metadb.dlm.DlmTableUnitRepository; -import com.oceanbase.odc.metadb.dlm.TaskGeneratorEntity; -import com.oceanbase.odc.metadb.dlm.TaskGeneratorRepository; -import com.oceanbase.odc.metadb.dlm.TaskUnitEntity; -import com.oceanbase.odc.metadb.dlm.TaskUnitRepository; -import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; -import com.oceanbase.odc.service.dlm.utils.DlmJobIdUtil; -import com.oceanbase.odc.service.dlm.utils.TaskGeneratorMapper; -import com.oceanbase.odc.service.dlm.utils.TaskUnitMapper; -import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; -import com.oceanbase.tools.migrator.common.dto.JobStatistic; -import com.oceanbase.tools.migrator.common.dto.TableSizeInfo; -import com.oceanbase.tools.migrator.common.dto.TaskGenerator; -import com.oceanbase.tools.migrator.common.meta.TableMeta; -import com.oceanbase.tools.migrator.core.IJobStore; -import com.oceanbase.tools.migrator.core.meta.ClusterMeta; -import com.oceanbase.tools.migrator.core.meta.JobMeta; -import com.oceanbase.tools.migrator.core.meta.TaskMeta; -import com.oceanbase.tools.migrator.core.meta.TenantMeta; - -import lombok.extern.slf4j.Slf4j; - -/** - * @Author：tinker - * @Date: 2023/5/8 19:27 - * @Descripition: TODO Store runtime data and use it to resume execution from a breakpoint. - */ -@Component -@Slf4j -public class DataArchiveJobStore implements IJobStore { - - @Value("${odc.task.dlm.support-breakpoint-recovery:false}") - private boolean supportBreakpointRecovery; - @Autowired - private DlmLimiterService limiterService; - @Autowired - private TaskGeneratorRepository taskGeneratorRepository; - @Autowired - private TaskUnitRepository taskUnitRepository; - @Autowired - private DlmTableUnitRepository dlmTableUnitRepository; - - private final TaskGeneratorMapper taskGeneratorMapper = TaskGeneratorMapper.INSTANCE; - private final TaskUnitMapper taskUnitMapper = TaskUnitMapper.INSTANCE; - - @Override - public TaskGenerator getTaskGenerator(String generatorId, String jobId) { - if (supportBreakpointRecovery) { - return taskGeneratorRepository.findByJobId(jobId).map(taskGeneratorMapper::entityToModel) - .orElse(null); - } - return null; - } - - @Override - public void storeTaskGenerator(TaskGenerator taskGenerator) { - if (supportBreakpointRecovery) { - Optional optional = taskGeneratorRepository.findByGeneratorId(taskGenerator.getId()); - TaskGeneratorEntity entity; - if (optional.isPresent()) { - entity = optional.get(); - entity.setStatus(taskGenerator.getGeneratorStatus().name()); - entity.setTaskCount(taskGenerator.getTaskCount()); - entity.setPartitionSavePoint(taskGenerator.getGeneratorPartitionSavepoint()); - entity.setProcessedRowCount(taskGenerator.getProcessedRowCount()); - entity.setProcessedDataSize(taskGenerator.getProcessedDataSize()); - if (taskGenerator.getGeneratorSavePoint() != null) { - entity.setPrimaryKeySavePoint(taskGenerator.getGeneratorSavePoint().toSqlString()); - } - } else { - entity = taskGeneratorMapper.modelToEntity(taskGenerator); - } - taskGeneratorRepository.save(entity); - } - } - - @Override - public void bindGeneratorToJob(String jobId, TaskGenerator taskGenerator) {} - - @Override - public JobStatistic getJobStatistic(String s) { - return new JobStatistic(); - } - - @Override - public void storeJobStatistic(JobMeta jobMeta) { - DlmTableUnitStatistic dlmExecutionDetail = new DlmTableUnitStatistic(); - dlmExecutionDetail.setProcessedRowCount(jobMeta.getJobStat().getRowCount()); - dlmExecutionDetail.setProcessedRowsPerSecond(jobMeta.getJobStat().getAvgRowCount()); - dlmExecutionDetail.setReadRowCount(jobMeta.getJobStat().getReadRowCount()); - dlmExecutionDetail.setReadRowsPerSecond(jobMeta.getJobStat().getAvgReadRowCount()); - dlmTableUnitRepository.updateStatisticByDlmTableUnitId(jobMeta.getJobId(), - JsonUtils.toJson(dlmExecutionDetail)); - } - - @Override - public List getTaskMeta(JobMeta jobMeta) { - if (supportBreakpointRecovery) { - List tasks = taskUnitRepository.findByGeneratorId(jobMeta.getGenerator().getId()).stream().map( - taskUnitMapper::entityToModel).collect( - Collectors.toList()); - tasks.forEach(o -> o.setJobMeta(jobMeta)); - return tasks; - } - return null; - } - - @Override - public void storeTaskMeta(TaskMeta taskMeta) { - if (supportBreakpointRecovery) { - Optional optional = taskUnitRepository.findByJobIdAndGeneratorIdAndTaskIndex( - taskMeta.getJobMeta().getJobId(), taskMeta.getGeneratorId(), taskMeta.getTaskIndex()); - TaskUnitEntity entity; - if (optional.isPresent()) { - entity = optional.get(); - entity.setStatus(taskMeta.getTaskStatus().name()); - entity.setPartitionName(taskMeta.getPartitionName()); - if (taskMeta.getMinPrimaryKey() != null) { - entity.setLowerBoundPrimaryKey(taskMeta.getMinPrimaryKey().toSqlString()); - } - if (taskMeta.getMaxPrimaryKey() != null) { - entity.setUpperBoundPrimaryKey(taskMeta.getMaxPrimaryKey().toSqlString()); - } - if (taskMeta.getCursorPrimaryKey() != null) { - entity.setPrimaryKeyCursor(taskMeta.getCursorPrimaryKey().toSqlString()); - } - } else { - entity = taskUnitMapper.modelToEntity(taskMeta); - } - taskUnitRepository.save(entity); - } - } - - @Override - public Long getAbnormalTaskIndex(String jobId) { - if (supportBreakpointRecovery) { - Long abnormalTaskCount = taskUnitRepository.findAbnormalTaskByJobId(jobId); - if (abnormalTaskCount != 0) { - return abnormalTaskCount; - } - } - return null; - } - - @Override - public void updateTableSizeInfo(TableSizeInfo tableSizeInfo, long l) { - - } - - @Override - public void updateLimiter(JobMeta jobMeta) { - RateLimitConfiguration rateLimit; - try { - rateLimit = limiterService - .getByOrderIdOrElseDefaultConfig(Long.parseLong(DlmJobIdUtil.getJobName(jobMeta.getJobId()))); - } catch (Exception e) { - log.warn("Update limiter failed,jobId={},error={}", - jobMeta.getJobId(), e); - return; - } - setClusterLimitConfig(jobMeta.getSourceCluster(), rateLimit.getDataSizeLimit()); - setClusterLimitConfig(jobMeta.getTargetCluster(), rateLimit.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getSourceTenant(), rateLimit.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getTargetTenant(), rateLimit.getDataSizeLimit()); - setTableLimitConfig(jobMeta.getSourceTableMeta(), rateLimit.getRowLimit()); - setTableLimitConfig(jobMeta.getTargetTableMeta(), rateLimit.getRowLimit()); - } - - private void setClusterLimitConfig(ClusterMeta clusterMeta, long dataSizeLimit) { - clusterMeta.setReadSizeLimit(dataSizeLimit); - clusterMeta.setWriteSizeLimit(dataSizeLimit); - clusterMeta.setWriteUsedQuota(1); - clusterMeta.setReadUsedQuota(1); - } - - private void setTenantLimitConfig(TenantMeta tenantMeta, long dataSizeLimit) { - tenantMeta.setReadSizeLimit(dataSizeLimit); - tenantMeta.setWriteSizeLimit(dataSizeLimit); - tenantMeta.setWriteUsedQuota(1); - tenantMeta.setReadUsedQuota(1); - } - - private void setTableLimitConfig(TableMeta tableMeta, int rowLimit) { - tableMeta.setReadRowCountLimit(rowLimit); - tableMeta.setWriteRowCountLimit(rowLimit); - } - -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java index 56e8a383be..e33817c7f0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java @@ -27,7 +27,8 @@ import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; import com.oceanbase.tools.migrator.common.configure.DataSourceInfo; -import com.oceanbase.tools.migrator.common.enums.DataBaseType; +import com.oceanbase.tools.migrator.common.enums.DatasourceType; +import com.oceanbase.tools.migrator.datasource.fs.FileFormat; import lombok.extern.slf4j.Slf4j; @@ -43,29 +44,14 @@ public static ConnectionConfig toConnectionConfig(DataSourceInfo dataSourceInfo) ConnectionConfig connectionConfig = new ConnectionConfig(); connectionConfig.setDefaultSchema(dataSourceInfo.getDatabaseName()); connectionConfig.setPassword(dataSourceInfo.getPassword()); - connectionConfig.setHost(dataSourceInfo.getIp()); + connectionConfig.setHost(dataSourceInfo.getHost()); connectionConfig.setPort(dataSourceInfo.getPort()); - connectionConfig.setUsername(dataSourceInfo.getFullUserName()); - connectionConfig.setType(ConnectType.valueOf(dataSourceInfo.getDatabaseType().name())); - // convert full username to native user name - if (dataSourceInfo.getDatabaseType() == DataBaseType.OB_ORACLE) { - String userName = connectionConfig.getUsername(); - if (userName.contains("#")) { - userName = userName.split("#")[0]; - } - if (userName.contains("@")) { - userName = userName.split("@")[0]; - } - if (userName.contains("\"")) { - userName = userName.replace("\"", ""); - } - connectionConfig.setUsername(userName); - connectionConfig.setTenantName(dataSourceInfo.getTenantName()); - connectionConfig.setClusterName(dataSourceInfo.getClusterName()); - } + connectionConfig.setUsername(dataSourceInfo.getUsername()); + connectionConfig.setType(ConnectType.valueOf(dataSourceInfo.getType().name())); return connectionConfig; } + public static DataSourceInfo toDataSourceInfo(ConnectionConfig connectionConfig, String schemaName) { DataSourceInfo dataSourceInfo = new DataSourceInfo(); dataSourceInfo.setDatabaseName(connectionConfig.getDefaultSchema()); @@ -73,40 +59,45 @@ public static DataSourceInfo toDataSourceInfo(ConnectionConfig connectionConfig, if (StringUtils.isNotEmpty(connectionConfig.getPassword())) { dataSourceInfo.setPassword(connectionConfig.getPassword()); } - dataSourceInfo.setIp(connectionConfig.getHost()); + dataSourceInfo.setHost(connectionConfig.getHost()); dataSourceInfo.setPort(connectionConfig.getPort()); switch (connectionConfig.getDialectType()) { case DORIS: case MYSQL: { - dataSourceInfo.setFullUserName(connectionConfig.getUsername()); - dataSourceInfo.setDatabaseType(DataBaseType.MYSQL); + dataSourceInfo.setUsername(connectionConfig.getUsername()); + dataSourceInfo.setType(DatasourceType.MYSQL); break; } case OB_MYSQL: { dataSourceInfo - .setFullUserName(OBConsoleDataSourceFactory.getUsername(connectionConfig)); - dataSourceInfo.setDatabaseType(DataBaseType.OB_MYSQL); - dataSourceInfo.setClusterName(connectionConfig.getClusterName()); - dataSourceInfo.setSysDatabaseName("oceanbase"); + .setUsername(OBConsoleDataSourceFactory.getUsername(connectionConfig)); + dataSourceInfo.setType(DatasourceType.OB_MYSQL); break; } case OB_ORACLE: - dataSourceInfo.setFullUserName(OBConsoleDataSourceFactory.getUsername(connectionConfig)); - dataSourceInfo.setClusterName(connectionConfig.getClusterName()); - dataSourceInfo.setTenantName(connectionConfig.getTenantName()); - dataSourceInfo.setDatabaseType(DataBaseType.OB_ORACLE); + dataSourceInfo.setUsername(OBConsoleDataSourceFactory.getUsername(connectionConfig)); + dataSourceInfo.setType(DatasourceType.OB_ORACLE); break; case POSTGRESQL: - dataSourceInfo.setFullUserName(connectionConfig.getUsername()); + dataSourceInfo.setUsername(connectionConfig.getUsername()); connectionConfig.setDefaultSchema(schemaName); String jdbcUrl = getJdbcUrl(connectionConfig) + "&stringtype=unspecified"; - dataSourceInfo.setJdbcUrl(jdbcUrl); - dataSourceInfo.setDatabaseType(DataBaseType.POSTGRESQL); + dataSourceInfo.setUrl(jdbcUrl); + dataSourceInfo.setType(DatasourceType.POSTGRESQL); break; case ORACLE: - dataSourceInfo.setJdbcUrl(getJdbcUrl(connectionConfig)); - dataSourceInfo.setDatabaseType(DataBaseType.ORACLE); - dataSourceInfo.setFullUserName(getOracleUsername(connectionConfig)); + dataSourceInfo.setUrl(getJdbcUrl(connectionConfig)); + dataSourceInfo.setType(DatasourceType.ORACLE); + dataSourceInfo.setUsername(getOracleUsername(connectionConfig)); + break; + case FILE_SYSTEM: + dataSourceInfo.setHost(connectionConfig.getHost()); + dataSourceInfo.setType(DatasourceType.valueOf(connectionConfig.getType().name())); + dataSourceInfo.setUsername(connectionConfig.getUsername()); + dataSourceInfo.setPassword(connectionConfig.getPassword()); + dataSourceInfo.setFileFormat(FileFormat.CSV); + dataSourceInfo.setRegion(connectionConfig.getRegion()); + dataSourceInfo.setDefaultCharset("UTF-8"); break; default: log.warn(String.format("Unsupported datasource type:%s", connectionConfig.getDialectType())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java index b852f42cdb..06103cadc9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java @@ -15,8 +15,10 @@ */ package com.oceanbase.odc.service.dlm.model; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import com.oceanbase.odc.common.util.StringUtils; @@ -40,6 +42,14 @@ public class DataArchiveTableConfig { // the sql condition such as "gmt_create < '2023-01-01'" private String conditionExpression; + private String minKey; + + private String maxKey; + + private Map partName2MinKey = new HashMap<>(); + + private Map partName2MaxKey = new HashMap<>(); + public String getTargetTableName() { return StringUtils.isEmpty(targetTableName) ? tableName : targetTableName; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java index 2370ed086d..03a0b03183 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java @@ -16,11 +16,14 @@ package com.oceanbase.odc.service.dlm.model; import java.util.Date; +import java.util.Set; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.migrator.common.configure.DataSourceInfo; import com.oceanbase.tools.migrator.common.enums.JobType; +import com.oceanbase.tools.migrator.limiter.LimiterConfig; import lombok.Data; @@ -48,6 +51,10 @@ public class DlmTableUnit { private DataSourceInfo targetDatasourceInfo; + private LimiterConfig sourceLimitConfig; + + private LimiterConfig targetLimitConfig; + private DlmTableUnitStatistic statistic; private DlmTableUnitParameters parameters; @@ -60,4 +67,6 @@ public class DlmTableUnit { private Date endTime; + private Set syncTableStructure; + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java index 9c61909e5a..8e124d27e7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java @@ -36,16 +36,15 @@ public interface TaskGeneratorMapper { @Mapping(source = "generatorId", target = "id") @Mapping(source = "status", target = "generatorStatus") - @Mapping(target = "generatorType", constant = "AUTO") - @Mapping(source = "partitionSavePoint", target = "generatorPartitionSavepoint") - @Mapping(target = "generatorSavePoint", + @Mapping(source = "partitionSavePoint", target = "partitionSavePoint") + @Mapping(target = "primaryKeySavePoint", expression = "java(com.oceanbase.tools.migrator.common.element.PrimaryKey.valuesOf(entity.getPrimaryKeySavePoint()))") TaskGenerator entityToModel(TaskGeneratorEntity entity); @InheritInverseConfiguration @Mapping(target = "type", constant = "AUTO") @Mapping(target = "primaryKeySavePoint", - expression = "java(model.getGeneratorSavePoint() != null ?model.getGeneratorSavePoint().toSqlString():null)") + expression = "java(model.getPrimaryKeySavePoint() != null ?model.getPrimaryKeySavePoint().toSqlString():null)") @Mapping(target = "id", ignore = true) TaskGeneratorEntity modelToEntity(TaskGenerator model); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java index 0fa3896d45..256fef7270 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java @@ -44,9 +44,6 @@ import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.partitionplan.model.PartitionPlanConfig; import com.oceanbase.odc.service.quartz.util.QuartzCronExpressionUtils; -import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleParameters; -import com.oceanbase.odc.service.schedule.model.JobType; -import com.oceanbase.odc.service.schedule.model.OperationType; import com.oceanbase.odc.service.schedule.model.TriggerConfig; import com.oceanbase.odc.service.schedule.model.TriggerStrategy; @@ -74,8 +71,6 @@ public class CreateFlowInstanceProcessAspect implements InitializingBean { @Value("${odc.task.trigger.minimum-interval:600}") private Long triggerMinimumIntervalSeconds; - private final Map scheduleTaskPreprocessors = new HashMap<>(); - private final Map flowTaskPreprocessors = new HashMap<>(); @Pointcut("@annotation(com.oceanbase.odc.service.flow.processor.EnablePreprocess) && args(com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq)") @@ -148,14 +143,6 @@ private void validateTriggerConfig(CreateFlowInstanceReq req) { if (parameters.getDroppingTrigger() != null) { validateTriggerConfig(parameters.getDroppingTrigger()); } - return; - } - if (req.getParameters() instanceof AlterScheduleParameters) { - AlterScheduleParameters parameters = (AlterScheduleParameters) req.getParameters(); - if (parameters.getOperationType() == OperationType.CREATE - || parameters.getOperationType() == OperationType.UPDATE) { - validateTriggerConfig(parameters.getTriggerConfig()); - } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java index 02898febbf..48724627a0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java @@ -302,13 +302,13 @@ public long calculatePartSize(long fileLength) { * 也就是杭州的client只允许操作杭州的bucket，不允许跨域操作 */ private void validateBucket() { - if (objectStorageConfiguration.getCloudProvider() != CloudProvider.ALIBABA_CLOUD) { - return; - } String bucketName = getBucketName(); boolean isExist = publicEndpointCloudObjectStorage.doesBucketExist(bucketName); Verify.verify(isExist, String.format("object storage bucket '%s' not exists", bucketName)); + if (objectStorageConfiguration.getCloudProvider() != CloudProvider.ALIBABA_CLOUD) { + return; + } String region = objectStorageConfiguration.getRegion(); if (StringUtils.isNotEmpty(region)) { String location = publicEndpointCloudObjectStorage.getBucketLocation(bucketName); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java index 60d75792f4..617c40d956 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.objectstorage.cloud; import java.io.File; +import java.io.InputStream; import java.net.URL; import java.util.Date; import java.util.List; @@ -61,6 +62,9 @@ public interface CloudObjectStorage { PutObjectResult putObject(String bucketName, String key, File file, ObjectMetadata metadata) throws CloudException; + PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException; + default PutObjectResult putObject(String bucketName, String key, File file) { return putObject(bucketName, key, file, null); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java index 059a7ba14e..acd9c43aea 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java @@ -170,6 +170,23 @@ public PutObjectResult putObject(String bucketName, String key, File file, Objec return putObject; } + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException { + PutObjectResult putObject = callOssMethod("Put object", () -> { + com.aliyun.oss.model.ObjectMetadata objectMetadata = toOss(metadata); + com.aliyun.oss.model.PutObjectResult ossResult = oss.putObject(bucketName, key, in, objectMetadata); + PutObjectResult result = new PutObjectResult(); + result.setVersionId(ossResult.getVersionId()); + result.setETag(ossResult.getETag()); + result.setRequestId(ossResult.getRequestId()); + result.setClientCRC(ossResult.getClientCRC()); + result.setServerCRC(ossResult.getServerCRC()); + return result; + }); + return putObject; + } + @Override public CopyObjectResult copyObject(String bucketName, String from, String to) throws CloudException { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java index e8e80d226b..d5f6527a0f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java @@ -183,6 +183,23 @@ public PutObjectResult putObject(String bucketName, String key, File file, Objec }); } + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException { + return callAmazonMethod("Put object", () -> { + com.amazonaws.services.s3.model.ObjectMetadata objectMetadata = toS3(metadata); + PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, in, objectMetadata); + if (metadata.hasTag()) { + putRequest.withTagging(toS3(metadata.getTagging())); + } + com.amazonaws.services.s3.model.PutObjectResult s3Result = s3.putObject(putRequest); + PutObjectResult result = new PutObjectResult(); + result.setVersionId(s3Result.getVersionId()); + result.setETag(s3Result.getETag()); + return result; + }); + } + @Override public CopyObjectResult copyObject(String bucketName, String from, String to) throws CloudException { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java index 7accc9b39b..91dec37407 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.objectstorage.cloud.client; import java.io.File; +import java.io.InputStream; import java.net.URL; import java.util.Date; import java.util.List; @@ -77,6 +78,12 @@ public PutObjectResult putObject(String bucketName, String key, File file, Objec throw new UnsupportedException(); } + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException { + throw new UnsupportedException(); + } + @Override public CopyObjectResult copyObject(String bucketName, String from, String to) throws CloudException { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java index 661574d65c..d37fb4ecef 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java @@ -27,7 +27,10 @@ import org.quartz.JobKey; import org.quartz.UnableToInterruptJobException; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.core.shared.exception.UnsupportedException; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; import com.oceanbase.odc.service.common.util.SpringContextUtil; import com.oceanbase.odc.service.monitor.DefaultMeterName; import com.oceanbase.odc.service.monitor.MeterKey; @@ -75,6 +78,16 @@ public void execute(JobExecutionContext context) throws JobExecutionException { sendEndMetric(); } catch (Exception e) { sendFailedMetric(); + try { + log.info("Start to update schedule task status to failed,jobKey={}", jobKey); + ScheduleTaskRepository taskRepository = SpringContextUtil.getBean(ScheduleTaskRepository.class); + ScheduleTaskEntity taskEntity = (ScheduleTaskEntity) context.getResult(); + if (taskEntity != null && taskEntity.getId() != null) { + taskRepository.updateStatusById(taskEntity.getId(), TaskStatus.FAILED); + } + } catch (Exception innerException) { + log.warn("Update schedule task status failed.", innerException); + } log.warn("Job execute failed,job key={},fire time={}.", context.getJobDetail().getKey(), context.getFireTime(), e); } finally { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index e6503004bb..cab45c4c98 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -42,6 +42,7 @@ import org.quartz.Trigger; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.cglib.beans.BeanMap; import org.springframework.context.annotation.Lazy; import org.springframework.core.io.InputStreamResource; @@ -151,6 +152,9 @@ @Service @SkipAuthorize public class ScheduleService { + + @Value("${odc.task.trigger.minimum-interval:600}") + private int minInterval; @Autowired private ScheduleRepository scheduleRepository; @@ -237,6 +241,7 @@ public List dispatchCreateSchedule(CreateFlowInstanceReq ScheduleChangeParams scheduleChangeParams; switch (parameters.getOperationType()) { case CREATE: { + validateTriggerConfig(parameters.getTriggerConfig()); CreateScheduleReq createScheduleReq = new CreateScheduleReq(); createScheduleReq.setParameters(parameters.getScheduleTaskParameters()); createScheduleReq.setTriggerConfig(parameters.getTriggerConfig()); @@ -274,7 +279,6 @@ public ChangeScheduleResp changeSchedule(ScheduleChangeParams req) { // create or load target schedule if (req.getOperationType() == OperationType.CREATE) { PreConditions.notNull(req.getCreateScheduleReq(), "req.createScheduleReq"); - validateTriggerConfig(req.getCreateScheduleReq().getTriggerConfig()); ScheduleEntity entity = new ScheduleEntity(); entity.setName(req.getCreateScheduleReq().getName()); @@ -420,10 +424,8 @@ private void validateTriggerConfig(TriggerConfig triggerConfig) { throw new IllegalArgumentException("Invalid cron expression"); } long intervalMills = nextFiveFireTimes.get(1).getTime() - nextFiveFireTimes.get(0).getTime(); - if (intervalMills / 1000 < 10 * 60) { - throw new IllegalArgumentException( - "The interval between weeks is too short. The minimum interval is 10 minutes."); - } + PreConditions.validArgumentState(intervalMills / 1000 > minInterval, ErrorCodes.ScheduleIntervalTooShort, + new Object[] {minInterval}, null); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java index e3216a388e..98c202020e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; import org.quartz.JobExecutionContext; @@ -29,12 +30,14 @@ import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.dlm.DLMConfiguration; +import com.oceanbase.odc.service.dlm.DLMService; import com.oceanbase.odc.service.dlm.DataSourceInfoMapper; import com.oceanbase.odc.service.dlm.DlmLimiterService; +import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.odc.service.quartz.util.ScheduleTaskUtils; import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.service.task.base.dataarchive.DataArchiveTask; -import com.oceanbase.odc.service.task.config.TaskFrameworkEnabledProperties; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.executor.task.TaskDescription; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; @@ -58,24 +61,23 @@ public abstract class AbstractDlmJob implements OdcJob { public final DatabaseService databaseService; public final ScheduleService scheduleService; public final DlmLimiterService limiterService; + public final DLMService dlmService; public JobScheduler jobScheduler = null; - - public final TaskFrameworkEnabledProperties taskFrameworkProperties; - public final TaskFrameworkService taskFrameworkService; + public final DLMConfiguration dlmConfiguration; + public AbstractDlmJob() { scheduleTaskRepository = SpringContextUtil.getBean(ScheduleTaskRepository.class); databaseService = SpringContextUtil.getBean(DatabaseService.class); scheduleService = SpringContextUtil.getBean(ScheduleService.class); limiterService = SpringContextUtil.getBean(DlmLimiterService.class); - taskFrameworkProperties = SpringContextUtil.getBean(TaskFrameworkEnabledProperties.class); taskFrameworkService = SpringContextUtil.getBean(TaskFrameworkService.class); - if (taskFrameworkProperties.isEnabled()) { - jobScheduler = SpringContextUtil.getBean(JobScheduler.class); - } + jobScheduler = SpringContextUtil.getBean(JobScheduler.class); + dlmService = SpringContextUtil.getBean(DLMService.class); + dlmConfiguration = SpringContextUtil.getBean(DLMConfiguration.class); } public DataSourceInfo getDataSourceInfo(Long databaseId) { @@ -83,6 +85,8 @@ public DataSourceInfo getDataSourceInfo(Long databaseId) { ConnectionConfig config = databaseService.findDataSourceForTaskById(databaseId); DataSourceInfo dataSourceInfo = DataSourceInfoMapper.toDataSourceInfo(config, db.getName()); dataSourceInfo.setDatabaseName(db.getName()); + dataSourceInfo.setSessionLimitRatio(dlmConfiguration.getSessionLimitingRatio()); + dataSourceInfo.setEnabledLimit(dlmConfiguration.isSessionLimitingEnabled()); return dataSourceInfo; } @@ -129,10 +133,25 @@ public Long publishJob(DLMJobReq parameters, Long timeoutMillis, CloudProvider p } public DLMJobReq getDLMJobReq(Long jobId) { - return JsonUtils.fromJson(JsonUtils.fromJson( + DLMJobReq dlmJobReq = JsonUtils.fromJson(JsonUtils.fromJson( taskFrameworkService.find(jobId).getJobParametersJson(), new TypeReference>() {}).get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), DLMJobReq.class); + Map tableName2Unit = + dlmService.findByScheduleTaskId(dlmJobReq.getScheduleTaskId()).stream() + .collect( + Collectors.toMap(DlmTableUnit::getTableName, o -> o)); + dlmJobReq.getTables().forEach(o -> { + if (tableName2Unit.containsKey(o.getTableName()) + && tableName2Unit.get(o.getTableName()).getStatistic() != null) { + o.setPartName2MinKey(tableName2Unit.get(o.getTableName()).getStatistic().getPartName2MinKey()); + o.setPartName2MaxKey(tableName2Unit.get(o.getTableName()).getStatistic().getPartName2MaxKey()); + o.setMinKey(tableName2Unit.get(o.getTableName()).getStatistic().getGlobalMinKey()); + o.setMaxKey(tableName2Unit.get(o.getTableName()).getStatistic().getGlobalMaxKey()); + } + }); + + return dlmJobReq; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java index be4bd69f4f..c0be02423a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java @@ -65,6 +65,7 @@ public void executeJob(JobExecutionContext context) { DLMJobReq parameters = getDLMJobReq(dataArchiveTask.getJobId()); parameters.setJobType(JobType.DELETE); + parameters.setFireTime(context.getFireTime()); parameters.setScheduleTaskId(taskEntity.getId()); parameters .setRateLimit(limiterService.getByOrderIdOrElseDefaultConfig(Long.parseLong(taskEntity.getJobName()))); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java index ef6b448532..6534f436aa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java @@ -48,6 +48,7 @@ private void executeInTaskFramework(JobExecutionContext context) { parameters.setScheduleTaskId(taskEntity.getId()); parameters.setJobType(JobType.MIGRATE); parameters.setTables(dataArchiveParameters.getTables()); + parameters.setFireTime(context.getFireTime()); for (DataArchiveTableConfig tableConfig : parameters.getTables()) { tableConfig.setConditionExpression(StringUtils.isNotEmpty(tableConfig.getConditionExpression()) ? DataArchiveConditionUtil.parseCondition(tableConfig.getConditionExpression(), diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java index 67ef30c5d8..dc139884da 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java @@ -63,6 +63,7 @@ public void executeJob(JobExecutionContext context) { DataSourceInfo tempDataSource = parameters.getSourceDs(); parameters.setSourceDs(parameters.getTargetDs()); parameters.setTargetDs(tempDataSource); + parameters.setFireTime(context.getFireTime()); parameters .setRateLimit(limiterService.getByOrderIdOrElseDefaultConfig(Long.parseLong(taskEntity.getJobName()))); parameters.getTables().forEach(o -> { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java index 392abd3f72..00e2a60fc6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java @@ -72,6 +72,7 @@ private void executeInTaskFramework(JobExecutionContext context) { parameters.getSourceDs().setQueryTimeout(dataDeleteParameters.getQueryTimeout()); parameters.getTargetDs().setQueryTimeout(dataDeleteParameters.getQueryTimeout()); parameters.setShardingStrategy(dataDeleteParameters.getShardingStrategy()); + parameters.setFireTime(context.getFireTime()); Long jobId = publishJob(parameters, dataDeleteParameters.getTimeoutMillis(), dataDeleteParameters.getDatabaseId()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java index 063accdf22..f6aa86192e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java @@ -15,6 +15,9 @@ */ package com.oceanbase.odc.service.schedule.model; +import java.util.HashMap; +import java.util.Map; + import lombok.Data; /** @@ -33,4 +36,12 @@ public class DlmTableUnitStatistic { private Long readRowsPerSecond = 0L; + private String globalMinKey; + + private String globalMaxKey; + + private Map partName2MinKey = new HashMap<>(); + + private Map partName2MaxKey = new HashMap<>(); + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java index b7bd32427e..352f267210 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java @@ -181,7 +181,7 @@ public void supportDataArchivingLink(ConnectionConfig sourceDs, ConnectionConfig targetDs.getRegion())); } if (sourceDs.getDialectType().isMysql()) { - if (!targetDs.getDialectType().isMysql()) { + if (!targetDs.getDialectType().isMysql() && targetDs.getDialectType() != DialectType.FILE_SYSTEM) { throw new UnsupportedException( String.format("Unsupported data link from %s to %s.", sourceDs.getDialectType(), targetDs.getDialectType())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java index 63c3b30f67..dad948983c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java @@ -18,18 +18,14 @@ import org.springframework.beans.factory.annotation.Autowired; import com.oceanbase.odc.core.session.ConnectionSession; -import com.oceanbase.odc.core.session.ConnectionSessionConstants; import com.oceanbase.odc.core.session.ConnectionSessionFactory; -import com.oceanbase.odc.core.shared.constant.DialectType; -import com.oceanbase.odc.core.shared.exception.UnsupportedException; -import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint; +import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.dlm.DLMConfiguration; -import com.oceanbase.odc.service.dlm.DLMTableStructureSynchronizer; import com.oceanbase.odc.service.dlm.model.DataArchiveParameters; -import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.schedule.model.OperationType; import com.oceanbase.odc.service.schedule.model.ScheduleChangeParams; import com.oceanbase.odc.service.schedule.model.ScheduleType; @@ -65,12 +61,15 @@ public void process(ScheduleChangeParams req) { Database sourceDb = databaseService.detail(parameters.getSourceDatabaseId()); Database targetDb = databaseService.detail(parameters.getTargetDataBaseId()); supportDataArchivingLink(sourceDb.getDataSource(), targetDb.getDataSource()); + if (!parameters.getSyncTableStructure().isEmpty()) { + PreConditions.validArgumentState(sourceDb.getDialectType() != targetDb.getDialectType(), + ErrorCodes.UnsupportedSyncTableStructure, + new Object[] {sourceDb.getDialectType(), targetDb.getDialectType()}, null); + } ConnectionConfig sourceDs = sourceDb.getDataSource(); sourceDs.setDefaultSchema(sourceDb.getName()); ConnectionSessionFactory sourceSessionFactory = new DefaultConnectSessionFactory(sourceDs); - ConnectionSessionFactory targetSessionFactory = new DefaultConnectSessionFactory(targetDb.getDataSource()); ConnectionSession sourceSession = sourceSessionFactory.generateSession(); - ConnectionSession targetSession = targetSessionFactory.generateSession(); try { if (parameters.isFullDatabase()) { parameters.setTables(getAllTables(sourceSession, sourceDb.getName())); @@ -78,36 +77,9 @@ public void process(ScheduleChangeParams req) { if (parameters.getTables().isEmpty()) { throw new IllegalArgumentException("The table list is empty."); } - DialectType sourceDbType = sourceSession.getDialectType(); - DialectType targetDbType = targetSession.getDialectType(); - InformationExtensionPoint sourceInformation = - ConnectionPluginUtil.getInformationExtension(sourceDbType); - InformationExtensionPoint targetInformation = - ConnectionPluginUtil.getInformationExtension(targetDbType); - String sourceDbVersion = - sourceSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY).execute( - sourceInformation::getDBVersion); - String targetDbVersion = - targetSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY).execute( - targetInformation::getDBVersion); - if (!parameters.getSyncTableStructure().isEmpty()) { - boolean supportedSyncTableStructure = DLMTableStructureSynchronizer.isSupportedSyncTableStructure( - sourceDbType, sourceDbVersion, targetDbType, targetDbVersion); - if (!supportedSyncTableStructure) { - log.warn( - "Synchronization of table structure is unsupported,sourceDbType={},sourceDbVersion={},targetDbType={},targetDbVersion={}", - sourceDbType, - sourceDbVersion, targetDbType, targetDbVersion); - throw new UnsupportedException(String.format( - "Synchronization of table structure is unsupported,sourceDbType=%s,sourceDbVersion=%s,targetDbType=%s,targetDbVersion=%s", - sourceDbType, - sourceDbVersion, targetDbType, targetDbVersion)); - } - } checkTableAndCondition(sourceSession, sourceDb, parameters.getTables(), parameters.getVariables()); } finally { sourceSession.expire(); - targetSession.expire(); } log.info("Data archive preprocessing has been completed."); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java index 6c7d63ec1d..1b934ab46f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -15,13 +15,11 @@ */ package com.oceanbase.odc.service.task.base.dataarchive; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import com.oceanbase.odc.common.json.JsonUtils; @@ -29,9 +27,9 @@ import com.oceanbase.odc.service.dlm.DLMJobFactory; import com.oceanbase.odc.service.dlm.DLMJobStore; import com.oceanbase.odc.service.dlm.DLMTableStructureSynchronizer; -import com.oceanbase.odc.service.dlm.DataSourceInfoMapper; import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.odc.service.dlm.model.DlmTableUnitParameters; +import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; import com.oceanbase.odc.service.dlm.utils.DlmJobIdUtil; import com.oceanbase.odc.service.schedule.job.DLMJobReq; import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; @@ -40,7 +38,9 @@ import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.util.JobUtils; import com.oceanbase.tools.migrator.common.enums.JobType; +import com.oceanbase.tools.migrator.core.meta.JobMeta; import com.oceanbase.tools.migrator.job.Job; +import com.oceanbase.tools.migrator.limiter.LimiterConfig; import com.oceanbase.tools.migrator.task.CheckMode; import lombok.extern.slf4j.Slf4j; @@ -58,7 +58,8 @@ public class DataArchiveTask extends TaskBase> { private DLMJobStore jobStore; private double progress = 0.0; private Job job; - private Map result; + private List toDoList; + private int currentIndex = 0; private boolean isToStop = false; public DataArchiveTask() {} @@ -67,97 +68,71 @@ public DataArchiveTask() {} protected void doInit(JobContext context) { jobStore = new DLMJobStore(JobUtils.getMetaDBConnectionConfig()); jobFactory = new DLMJobFactory(jobStore); - log.info("Init data-archive job env succeed,jobIdentity={}", context.getJobIdentity()); - } - - @Override - public boolean start() throws Exception { - - jobStore.setJobParameters(jobContext.getJobParameters()); - DLMJobReq parameters = - JsonUtils.fromJson( - jobContext.getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), - DLMJobReq.class); - if (parameters.getFireTime() == null) { - parameters.setFireTime(new Date()); - } try { - result = getDlmTableUnits(parameters).stream() - .collect(Collectors.toMap(DlmTableUnit::getDlmTableUnitId, o -> o)); - jobStore.setDlmTableUnits(result); + DLMJobReq parameters = + JsonUtils.fromJson( + jobContext.getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), + DLMJobReq.class); + initTableUnit(parameters); } catch (Exception e) { - log.warn("Get dlm job failed!", e); - context.getExceptionListener().onException(e); - return false; + log.warn("Initialization of the DLM job was failed,jobIdentity={}", context.getJobIdentity(), e); } - Set dlmTableUnitIds = result.keySet(); + log.info("Initialization of the DLM job was successful. Number of tables to be processed = {},jobIdentity={}", + toDoList.size(), context.getJobIdentity()); + } - for (String dlmTableUnitId : dlmTableUnitIds) { - DlmTableUnit dlmTableUnit = result.get(dlmTableUnitId); - if (isToStop) { - log.info("Job is terminated,jobIdentity={}", jobContext.getJobIdentity()); - break; - } + @Override + public boolean start() throws Exception { + while (!isToStop && currentIndex < toDoList.size()) { + DlmTableUnit dlmTableUnit = toDoList.get(currentIndex); if (dlmTableUnit.getStatus() == TaskStatus.DONE) { log.info("The table had been completed,tableName={}", dlmTableUnit.getTableName()); + currentIndex++; continue; } - startTableUnit(dlmTableUnitId); - if (parameters.getJobType() == JobType.MIGRATE) { - try { - DLMTableStructureSynchronizer.sync( - DataSourceInfoMapper.toConnectionConfig(parameters.getSourceDs()), - DataSourceInfoMapper.toConnectionConfig(parameters.getTargetDs()), - dlmTableUnit.getTableName(), dlmTableUnit.getTargetTableName(), - parameters.getSyncTableStructure()); - } catch (Exception e) { - log.warn("Failed to sync target table structure,table will be ignored,tableName={}", - dlmTableUnit.getTableName(), e); - if (!parameters.getSyncTableStructure().isEmpty()) { - finishTableUnit(dlmTableUnitId, TaskStatus.FAILED); - continue; - } - } - } + syncTableStructure(dlmTableUnit); try { + jobStore.setDlmTableUnit(dlmTableUnit); job = jobFactory.createJob(dlmTableUnit); - log.info("Init {} job succeed,DLMJobId={}", job.getJobMeta().getJobType(), job.getJobMeta().getJobId()); - log.info("{} job start,DLMJobId={}", job.getJobMeta().getJobType(), job.getJobMeta().getJobId()); - if (isToStop) { - finishTableUnit(dlmTableUnitId, TaskStatus.CANCELED); - job.stop(); - log.info("The task has stopped."); - break; - } else { - job.run(); - } - log.info("{} job finished,DLMJobId={}", dlmTableUnit.getType(), dlmTableUnitId); - finishTableUnit(dlmTableUnitId, TaskStatus.DONE); } catch (Throwable e) { - log.error("{} job failed,DLMJobId={},errorMsg={}", dlmTableUnit.getType(), dlmTableUnitId, e); - // set task status to failed if any job failed. - if (job != null && job.getJobMeta().isToStop()) { - finishTableUnit(dlmTableUnitId, TaskStatus.CANCELED); - } else { - finishTableUnit(dlmTableUnitId, TaskStatus.FAILED); - context.getExceptionListener().onException(e); - } + log.error("Failed to create job,dlmTableUnitId={}", dlmTableUnit.getDlmTableUnitId(), e); + dlmTableUnit.setStatus(isToStop ? TaskStatus.CANCELED : TaskStatus.FAILED); + currentIndex++; + continue; } + log.info("Init {} job succeed,dlmTableUnitId={}", dlmTableUnit.getType(), dlmTableUnit.getDlmTableUnitId()); + try { + dlmTableUnit.setStatus(TaskStatus.RUNNING); + dlmTableUnit.setStartTime(new Date()); + job.run(); + log.info("{} job finished,dlmTableUnitId={}", dlmTableUnit.getType(), dlmTableUnit.getDlmTableUnitId()); + dlmTableUnit.setStatus(TaskStatus.DONE); + } catch (Throwable e) { + dlmTableUnit.setStatus(isToStop ? TaskStatus.CANCELED : TaskStatus.FAILED); + context.getExceptionListener().onException(e); + } + dlmTableUnit.setEndTime(new Date()); + currentIndex++; } + log.info("All tables have been processed,jobIdentity={}.\n{}", jobContext.getJobIdentity(), buildReport()); return true; } - private void startTableUnit(String dlmTableUnitId) { - result.get(dlmTableUnitId).setStatus(TaskStatus.RUNNING); - result.get(dlmTableUnitId).setStartTime(new Date()); - } - - private void finishTableUnit(String dlmTableUnitId, TaskStatus status) { - result.get(dlmTableUnitId).setStatus(status); - result.get(dlmTableUnitId).setEndTime(new Date()); + private void syncTableStructure(DlmTableUnit tableUnit) { + if (tableUnit.getType() != JobType.MIGRATE) { + return; + } + try { + DLMTableStructureSynchronizer.sync(tableUnit.getSourceDatasourceInfo(), tableUnit.getTargetDatasourceInfo(), + tableUnit.getTableName(), tableUnit.getTargetTableName(), + tableUnit.getSyncTableStructure()); + } catch (Exception e) { + log.warn("Failed to sync target table structure,tableName={}", + tableUnit.getTableName(), e); + } } - private List getDlmTableUnits(DLMJobReq req) throws SQLException { + private void initTableUnit(DLMJobReq req) { List dlmTableUnits = new LinkedList<>(); req.getTables().forEach(table -> { DlmTableUnit dlmTableUnit = new DlmTableUnit(); @@ -171,6 +146,8 @@ private List getDlmTableUnits(DLMJobReq req) throws SQLException { jobParameter.setMigratePartitions(table.getPartitions()); jobParameter.setSyncDBObjectType(req.getSyncTableStructure()); jobParameter.setShardingStrategy(req.getShardingStrategy()); + jobParameter.setPartName2MinKey(table.getPartName2MinKey()); + jobParameter.setPartName2MaxKey(table.getPartName2MaxKey()); dlmTableUnit.setParameters(jobParameter); dlmTableUnit.setDlmTableUnitId(DlmJobIdUtil.generateHistoryJobId(req.getJobName(), req.getJobType().name(), req.getScheduleTaskId(), dlmTableUnits.size())); @@ -182,9 +159,37 @@ private List getDlmTableUnits(DLMJobReq req) throws SQLException { dlmTableUnit.setStatus(TaskStatus.PREPARING); dlmTableUnit.setType(req.getJobType()); dlmTableUnit.setStatistic(new DlmTableUnitStatistic()); + dlmTableUnit.setSyncTableStructure(req.getSyncTableStructure()); + LimiterConfig limiterConfig = new LimiterConfig(); + limiterConfig.setDataSizeLimit(req.getRateLimit().getDataSizeLimit()); + limiterConfig.setRowLimit(req.getRateLimit().getRowLimit()); + dlmTableUnit.setSourceLimitConfig(limiterConfig); + dlmTableUnit.setTargetLimitConfig(limiterConfig); dlmTableUnits.add(dlmTableUnit); }); - return dlmTableUnits; + toDoList = new LinkedList<>(dlmTableUnits); + } + + private String buildReport() { + StringBuilder sb = new StringBuilder(); + sb.append("Job report:\n"); + sb.append("Total tables: ").append(toDoList.size()).append("\n"); + sb.append("Success tables: ") + .append(toDoList.stream().filter(t -> t.getStatus() == TaskStatus.DONE).map(DlmTableUnit::getTableName) + .collect( + Collectors.joining(","))) + .append("\n"); + sb.append("Failed tables: ") + .append(toDoList.stream().filter(t -> t.getStatus() == TaskStatus.FAILED) + .map(DlmTableUnit::getTableName).collect( + Collectors.joining(","))) + .append("\n"); + sb.append("Canceled tables: ") + .append(toDoList.stream().filter(t -> t.getStatus() == TaskStatus.CANCELED) + .map(DlmTableUnit::getTableName).collect( + Collectors.joining(","))) + .append("\n"); + return sb.toString(); } @Override @@ -193,9 +198,9 @@ public void stop() throws Exception { if (job != null) { try { job.stop(); - result.forEach((k, v) -> { - if (!v.getStatus().isTerminated()) { - v.setStatus(TaskStatus.CANCELED); + toDoList.forEach(t -> { + if (!t.getStatus().isTerminated()) { + t.setStatus(TaskStatus.CANCELED); } }); } catch (Exception e) { @@ -214,13 +219,39 @@ public boolean modify(Map jobParameters) { if (!super.modify(jobParameters)) { return false; } - afterModifiedJobParameters(); + updateLimiter(jobParameters); return true; } - protected void afterModifiedJobParameters() { - if (jobStore != null) { - jobStore.setJobParameters(jobContext.getJobParameters()); + public void updateLimiter(Map jobParameters) { + if (job == null || job.getJobMeta() == null) { + return; + } + JobMeta jobMeta = job.getJobMeta(); + try { + RateLimitConfiguration params; + if (jobParameters.containsKey(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG)) { + params = JsonUtils.fromJson( + jobParameters.get(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG), + RateLimitConfiguration.class); + } else { + DLMJobReq dlmJobReq = JsonUtils.fromJson( + jobParameters.get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), + DLMJobReq.class); + params = dlmJobReq.getRateLimit(); + } + if (params.getDataSizeLimit() != null) { + jobMeta.getSourceLimiterConfig().setDataSizeLimit(params.getDataSizeLimit()); + jobMeta.getTargetLimiterConfig().setDataSizeLimit(params.getDataSizeLimit()); + log.info("Update rate limit success,dataSizeLimit={}", params.getDataSizeLimit()); + } + if (params.getRowLimit() != null) { + jobMeta.getSourceLimiterConfig().setRowLimit(params.getRowLimit()); + jobMeta.getTargetLimiterConfig().setRowLimit(params.getRowLimit()); + log.info("Update rate limit success,rowLimit={}", params.getRowLimit()); + } + } catch (Exception e) { + log.warn("Update rate limit failed,errorMsg={}", e.getMessage()); } } @@ -231,6 +262,6 @@ public double getProgress() { @Override public List getTaskResult() { - return new ArrayList<>(result.values()); + return new ArrayList<>(toDoList); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java index e61bd64b3f..50c428e1bb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java @@ -48,7 +48,7 @@ public class DLMResultProcessor extends DLMProcessorMatcher implements ResultPro @Override public void process(TaskResult result) { - log.info("Start refresh result,result={}", result.getResultJson()); + log.info("Start refresh result,jobIdentity={}", result.getJobIdentity()); try { List dlmTableUnits = JsonUtils.fromJson(result.getResultJson(), new TypeReference>() {}); diff --git a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java index aa20507701..e2230ee08f 100644 --- a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java +++ b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java @@ -84,6 +84,22 @@ public static TestResult initScriptFailed(Throwable throwable) { return fail(ErrorCodes.ConnectionInitScriptFailed, new String[] {message}); } + public static TestResult bucketNotExist(String bucketName) { + return fail(ErrorCodes.BucketNotExist, new String[] {bucketName}); + } + + public static TestResult invalidAccessKeyId(String accessKeyId) { + return fail(ErrorCodes.InvalidAccessKeyId, new String[] {accessKeyId}); + } + + public static TestResult akAccessDenied(String accessKeyId) { + return fail(ErrorCodes.AccessDenied, new String[] {accessKeyId}); + } + + public static TestResult signatureDoesNotMatch(String accessKeyId) { + return fail(ErrorCodes.SignatureDoesNotMatch, new String[] {accessKeyId}); + } + public static TestResult success() { TestResult result = new TestResult(); result.setActive(true); From 0fa667a351162b90719594901975b38299d231cb Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Tue, 14 Jan 2025 15:18:44 +0800 Subject: [PATCH 109/118] fix(session): killing sessions failed in OSC tasks #4166 --- .../odc/service/db/session/DefaultDBSessionManage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index b500e4bf8b..02d2f9032a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -454,7 +454,7 @@ private CompletableFuture doKillAllSessions(List list, Conne .forEach(sessionList -> { SessionExtensionPoint sessionExtension = ConnectionPluginUtil.getSessionExtension(connectionSession.getDialectType()); - Map connectionId2KillSql = sessionExtension.getKillQuerySqls( + Map connectionId2KillSql = sessionExtension.getKillSessionSqls( sessionList.stream().map(OdcDBSession::getSessionId).collect(Collectors.toSet())); doKill(connectionSession, connectionId2KillSql); }); From b4569948fbbb9d4337b38f0fabb0682862ea838d Mon Sep 17 00:00:00 2001 From: guowl3 Date: Tue, 14 Jan 2025 15:20:37 +0800 Subject: [PATCH 110/118] doc: init 4.3.3 changelog (#4162) * upgrade changelog * rsp comments * rsp comments * rsp comments * rsp comments * rsp comments * update changelog * rsp comments * rsp comments * rsp comments --- CHANGELOG-zh-CN.md | 94 +++++++++++++++++++++++++++++++++++++++++++ CHANGELOG.md | 99 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+) diff --git a/CHANGELOG-zh-CN.md b/CHANGELOG-zh-CN.md index 3dbf7d1847..6f5e3708c7 100644 --- a/CHANGELOG-zh-CN.md +++ b/CHANGELOG-zh-CN.md @@ -1,4 +1,98 @@ # OceanBase Developer Center (ODC) CHANGELOG +## 4.3.3 (2025-01-13) + +### 功能变化 + +数据生命周期管理 + +- 新增 Oracle 到对象存储的归档链路 +- 新增 MySQL 到对象存储的归档链路 +- 新增 OceanBase MySQL 到对象存储的归档链路 +- 新增 OceanBase Oracle 到对象存储的归档链路 +- 新增 PostgreSQL 到对象存储的归档链路 +- 支持回溯编辑历史，支持查看编辑内容前后对比 +- 支持定义动态目标表，解决按日、月等单独存放历史数据的诉求 +- 支持删除数据归档、清理任务，当任务已完成或已终止时支持对其进行删除操作 +- 优化回滚逻辑，仅回滚当次任务的归档数据 + +无锁结构变更 + +- 支持失败重试，为各环节可能导致失败的场景补充重试逻辑 +- 支持无锁结构变更状态展示，可以查看运行中任务进度 + +变更风险管控 + +- 增加全局项目角色，包括全局项目管理员、全局安全管理员以及全局 DBA +- 增加项目归档检测机制，归档前会检测项目中是否存在未结束的工单及周期任务 +- 支持删除项目，对于已归档的项目支持对其进行删除操作 +- 支持用户申请视图权限，对用户访问视图做了更细粒度的权限控制 +- SQL 窗口拓展了可执行的 SQL 类型，新支持 `call`、`comment`、`set session` 等类型 +- SQL 检查规范支持原生 Oracle 数据源 +- 支持原生 Oracle 数据源的变更走变更审批流程 +- 新增 2 条 SQL 检查规则，支持规范 `create like` 及 `create as` 建表语句 + +SQL 开发 + +- 支持 OceanBase 外表白屏化管理 +- 支持 OceanBase 分区表的二级分区展示 +- 支持编辑 OceanBase MySQL 模式的函数和存储过程 +- 支持通过 OBProxy 进行 PL 调试 + +其他 + +- 支持 SAML 的单点登录方式 +- 支持查杀原生 Oracle 数据源的会话 +- 适配 OceanBase 4.2.5、4.3.3 版本 +- 适配 OBKV SQL 模式 +- 启用 Secure Cookie 机制，加固数据传输安全 +- 平台表单（含工单列表、数据库列表）列宽支持拉伸 + +### 易用性改进 +- 支持固化项目搜索条件，避免频繁搜索高频操作项目 +- 支持用户登出再登入后仍旧可以定位在最近使用的项目下，简化用户操作路径 +- 风险识别规则中判断条件文案优化，统一采用运算符及英文表达，以避免歧义 +- 优化连接保活逻辑，每3分钟会主动发送一次数据库请求，保障连接的稳定性 +- 项目外工单模块增加项目列，方便用户快速识别工单所属项目 +- 除逻辑库变更, 分区计划, 影子表外，所有工单类型支持再次发起功能，再次发起后支持二次编辑工单参数 +- 工单可被管理及查看范围调整，管理员和 DBA 可管理项目内所有工单，其它角色仅可管理自己发起的工单。同时项目内所有成员均可查看项目内所有工单 + + +### 缺陷修复 + +数据源 + +- 堡垒机集成场景不会同步 `information_schema` 等内置数据库到项目内 +- 数据库同步异常挂起时无法恢复 + +工单 + +- 创建数据归档工单在个人空间仍产生审批流程 +- 数据归档/清理任务执行成功但执行记录状态异常 +- 非当前账号创建结构对比任务无法正常执行 +- Oracle 导出表结构存在虚拟列时导出会失败 +- OceanBase MySQL 源端库或目标库里若有一张表的 DDL 里指定全文索引的分词器，结构比对任务失败 +- 定时任务如果有太多的子任务，查看操作记录失败问题 +- 导出任务保留当前配置不生效 + +变更管控 + +- 没有导出权限也能导出视图 +- 分区计划无法禁用导致无法归档项目 + +SQL 开发 + +- SQL Check 特定场景下产生 NPE 异常 +- DROP PL 需要数据库变更的权限 +- 函数返回值类型为 Year 时无法正常显示 +- 当 PL 名称包含 @ 时 create 和 drop 语句将失败 +- 查看原生 Oracle 扩展了统计信息（`DBMS_STATS.CREATE_EXTENDED_STATS`）的表详情失败 +- 限制 SQL 影响的行数时，insert 语句不生效 +- 导出数组函数结果集时，空指针异常问题 +- 在 Chrome 118 版本的浏览器中，右键单击软件包子程序时没有运行按钮 +- 查看程序包包头中的子程序时报错 + +其他 +- 用户再次进入 ODC 时没有打开上次使用的项目 ## 4.3.2 (2024-09-27) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21dcf26463..f369e1c8c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,102 @@ +# OceanBase Developer Center (ODC) CHANGELOG + +## 4.3.3 (2025-01-13) + +### Feature Changes + +**Data Lifecycle Management** + +* Added archiving paths from Oracle to Object Storage. +* Added archiving paths from MySQL to Object Storage. +* Added archiving paths from OceanBase MySQL to Object Storage. +* Added archiving paths from OceanBase Oracle to Object Storage. +* Added archiving paths from PostgreSQL to Object Storage. +* Added support for editing history review with content comparison. +* Introduced dynamic target table definition to support storing historical data by day, month, or other time units. +* Added ability to delete archiving and cleanup tasks when completed or terminated. +* Optimized rollback logic to only revert data archived in the current task. + +**Online Schema Change** + +* Added retry mechanism with enhanced retry logic for various failure scenarios. +* Added status display for online schema changes, allowing progress tracking of running tasks. + +**Change Risk Management** + +* Introduced global project roles including Global Project Admin, Global Security Admin, and Global DBA. +* Added project archiving validation to check for unfinished tickets and periodic tasks before archiving. +* Added support for project deletion for archived projects. +* Implemented fine-grained view permission control with user-initiated permission requests. +* Expanded executable SQL types in SQL window to include `call`, `comment`, `set session`, and more. +* Extended SQL check support for native Oracle data sources. +* Added change approval workflow for native Oracle data sources. +* Added 2 new SQL check rules for standardizing `create like` and `create as` table creation statements. + +**SQL Development** + +* Added GUI support for OceanBase external tables. +* Added support for displaying secondary partitions in OceanBase partitioned tables. +* Added support for editing stored procedures in OceanBase MySQL mode. +* Added support for PL debugging via OBProxy. + +**Other Enhancements** + +* Added SAML single sign-on support. +* Added session kill capability for native Oracle data sources. +* Added compatibility for OceanBase V4.2.5 and V4.3.3. +* Added support for OBKV SQL mode. +* Enabled secure cookie mechanism for enhanced data transmission security. +* Added column width adjustment support for forms (ticket lists, database lists). + +### Usability Improvements + +* Added persistent project search criteria to reduce frequent searches. +* Maintained last used project context across user sessions. +* Standardized risk identification rule conditions using operators and English expressions to avoid ambiguity. +* Enhanced connection keep-alive logic with 3-minute database request intervals. +* Added project column in non-project ticket module for quick project identification. +* Extended "Create Again" functionality to all ticket types except logical database changes, partition plans, and shadow tables. +* Refined ticket management scope: admins and DBAs can manage all project tickets, while other roles can only manage self-initiated tickets. Also, all members can view tickets in their projects. + +### Bug Fixes + +**Data Sources** + +* Fixed synchronization of system databases like `information_schema` to projects in bastion host integration scenarios. +* Resolved database synchronization suspension issues. + +**Tickets** + +* Fixed approval workflow triggering in personal workspace for data archiving tickets. +* Fixed status inconsistency in data archiving/cleanup task execution records. +* Fixed structure comparison task execution issues for non-current account users. +* Fixed Oracle table structure export failures with virtual columns. +* Fixed structure comparison failures for OceanBase MySQL tables with full-text index tokenizers. +* Fixed operation record viewing failures for periodic tasks with numerous subtasks. +* Fixed non-working configuration retention in export tasks. + +**Change Management** + +* Fixed unauthorized view exports. +* Fixed partition plan disable issues affecting project archiving. + +**SQL Development** + +* Fixed NPE exceptions in specific SQL check scenarios. +* Fixed PL drop requiring database change permissions. +* Fixed display issues for functions with year return type. +* Fixed PL creation and drop failures with @ in names. +* Fixed table detail viewing failures for Oracle tables with extended statistics (`DBMS_STATS.CREATE_EXTENDED_STATS`). +* Fixed ineffective row limit for Insert statements. +* Fixed null pointer exceptions when exporting array function result sets. +* Fixed missing run button for package subprocedures in Chrome 118. +* Fixed error when viewing subprocedures in package headers. + +**Other** + +* Fixed last used project not opening on subsequent ODC access. + + ## 4.3.2 (2024-09-27) ### Feature Changes From e67a557e8bd680dc49b5512362e1e21d823b128b Mon Sep 17 00:00:00 2001 From: guowl3 Date: Wed, 15 Jan 2025 14:00:50 +0800 Subject: [PATCH 111/118] exclusion aws-java-sdk-bundle (#4167) --- pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pom.xml b/pom.xml index 92b55b325e..99c83c5f6e 100644 --- a/pom.xml +++ b/pom.xml @@ -168,6 +168,10 @@ org.slf4j slf4j-log4j12 + + com.amazonaws + aws-java-sdk-bundle + From 8971b2a7319da5be494ae13f5b71b7ef46d2a8e9 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Wed, 15 Jan 2025 15:56:19 +0800 Subject: [PATCH 112/118] fix(session): killing session may fail due to a wrong regex #4169 * fix wrong regex * add ut case * remove regex --- .../oceanbase/odc/common/util/HostUtils.java | 62 +++++++++++++++++++ .../odc/common/util/HostUtilsTest.java | 38 ++++++++++++ .../db/session/DefaultDBSessionManage.java | 44 +------------ 3 files changed, 103 insertions(+), 41 deletions(-) create mode 100644 server/odc-common/src/main/java/com/oceanbase/odc/common/util/HostUtils.java create mode 100644 server/odc-common/src/test/java/com/oceanbase/odc/common/util/HostUtilsTest.java diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/HostUtils.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/HostUtils.java new file mode 100644 index 0000000000..e0826b8b21 --- /dev/null +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/HostUtils.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.common.util; + +import javax.validation.constraints.NotNull; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +/** + * @Author: Lebie + * @Date: 2025/1/15 13:47 + * @Description: [] + */ +@Slf4j +public class HostUtils { + + @NotNull + public static ServerAddress extractServerAddress(String ipAndPort) { + String trimmed = StringUtils.trim(ipAndPort); + if (StringUtils.isBlank(trimmed)) { + log.info("unable to extract server address, text is empty"); + throw new IllegalArgumentException("Empty server address!"); + } + String[] segments = StringUtils.split(trimmed, ":"); + if (segments.length != 2) { + log.info("unable to extract server address, segments={}", segments); + throw new IllegalArgumentException("Invalid server address!"); + } + if (StringUtils.isEmpty(segments[0]) || StringUtils.isEmpty(segments[1])) { + log.info("unable to extract server address, segments={}", segments); + throw new IllegalArgumentException("Invalid server address!"); + } + return new ServerAddress(segments[0], segments[1]); + } + + + + @Data + public static class ServerAddress { + String ipAddress; + String port; + + public ServerAddress(String ipAddress, String port) { + this.ipAddress = ipAddress; + this.port = port; + } + } +} diff --git a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/HostUtilsTest.java b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/HostUtilsTest.java new file mode 100644 index 0000000000..14589c6d03 --- /dev/null +++ b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/HostUtilsTest.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.common.util; + +import org.junit.Test; + +import com.oceanbase.odc.common.util.HostUtils.ServerAddress; + +import junit.framework.Assert; + +public class HostUtilsTest { + @Test + public void testExtractServerAddress_ValidExpression() { + String ipAndPort = "1.1.1.1:1234"; + ServerAddress actual = HostUtils.extractServerAddress(ipAndPort); + Assert.assertEquals("1.1.1.1", actual.getIpAddress()); + Assert.assertEquals("1234", actual.getPort()); + } + + @Test(expected = IllegalArgumentException.class) + public void testExtractServerAddress_InvalidExpression() { + String ipAndPort = "1.1.1.1"; + HostUtils.extractServerAddress(ipAndPort); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index 02d2f9032a..7d0910c115 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -35,12 +35,8 @@ import java.util.concurrent.TimeoutException; import java.util.function.Predicate; import java.util.function.Supplier; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; -import javax.validation.constraints.NotNull; - import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.SetUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -48,6 +44,8 @@ import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; +import com.oceanbase.odc.common.util.HostUtils; +import com.oceanbase.odc.common.util.HostUtils.ServerAddress; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.VersionUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; @@ -82,9 +80,6 @@ @Slf4j public class DefaultDBSessionManage implements DBSessionManageFacade { - private static final String SERVER_REGEX = ".*(?([0-9]{1,3}.){1,3}([0-9]{1,3})):" - + "(?[0-9]{1,5}).*"; - private static final Pattern SERVER_PATTERN = Pattern.compile(SERVER_REGEX); private static final ConnectionMapper CONNECTION_MAPPER = ConnectionMapper.INSTANCE; private static final String GLOBAL_CLIENT_SESSION_OB_PROXY_VERSION_NUMBER = "4.2.3"; private static final String GLOBAL_CLIENT_SESSION_OB_VERSION_NUMBER = "4.2.5"; @@ -237,7 +232,7 @@ private List additionalKillIfNecessary(ConnectionSession conn Map sessionId2SvrAddr = getSessionList(connectionSession, s -> s.getSvrIp() != null) .stream().collect(Collectors.toMap(OdcDBSession::getSessionId, - s -> extractServerAddress(MoreObjects.firstNonNull(s.getSvrIp(), "")))); + s -> HostUtils.extractServerAddress(MoreObjects.firstNonNull(s.getSvrIp(), "")))); Map sqlId2SessionId = sqlTupleSessionIds.stream().collect( Collectors.toMap(s -> s.getSqlTuple().getSqlId(), SqlTupleSessionId::getSessionId)); @@ -424,28 +419,6 @@ private void directLinkServerAndExecute(String sql, ConnectionSession session, S } } - // extract text(query from the dictionary) to server address(ip, port) - // the text is expected be like 0.0.0.0:8888 - @NotNull - private ServerAddress extractServerAddress(String text) { - String trimmed = StringUtils.trim(text); - if (StringUtils.isBlank(trimmed)) { - log.info("unable to extract server address, text is empty"); - throw new IllegalStateException("Empty server address!"); - } - Matcher matcher = SERVER_PATTERN.matcher(trimmed); - if (!matcher.matches()) { - log.info("unable to extract server address, does not match pattern"); - throw new IllegalStateException("Invalid server address!"); - } - String ipAddress = matcher.group("ip"); - String port = matcher.group("port"); - if (StringUtils.isEmpty(ipAddress) || StringUtils.isEmpty(port)) { - log.info("unable to extract server address, ipAddress={}, port={}", ipAddress, port); - throw new IllegalStateException("Invalid server address!"); - } - return new ServerAddress(ipAddress, port); - } private CompletableFuture doKillAllSessions(List list, ConnectionSession connectionSession, Executor executor) { @@ -477,17 +450,6 @@ private void waitingForResult(Supplier> completableFutu } } - @Data - static class ServerAddress { - String ipAddress; - String port; - - public ServerAddress(String ipAddress, String port) { - this.ipAddress = ipAddress; - this.port = port; - } - } - @AllArgsConstructor @Data @NoArgsConstructor From 10a1f8636f42a84c476d9b88f8680941fb070d6d Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Thu, 16 Jan 2025 10:08:41 +0800 Subject: [PATCH 113/118] fix(session): get wrong server port #4171 --- .../stats/mysql/OBMySQLNoLessThan400StatsAccessor.java | 2 +- .../stats/oracle/OBOracleNoLessThan400StatsAccessor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java index a219659da4..35fe77a72b 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java @@ -41,7 +41,7 @@ public class OBMySQLNoLessThan400StatsAccessor extends OBMySQLStatsAccessor { + " STATE, " + " `USER_CLIENT_IP` as HOST, " + " HOST as PROXY_HOST, " - + " CONCAT(SVR_IP, ':', SVR_PORT) AS SVR_IP, " + + " CONCAT(SVR_IP, ':', SQL_PORT) AS SVR_IP, " + " TIME as EXECUTE_TIME, " + " CASE " + " WHEN `TRANS_STATE` IS NULL OR `TRANS_STATE` IN ('', 'IDLE', 'IN_TERMINATE', 'ABORTED', " diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java index cb727753a4..6ce9432dcf 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java @@ -38,7 +38,7 @@ public class OBOracleNoLessThan400StatsAccessor extends BaseOBOracleStatsAccesso + " STATE, " + " USER_CLIENT_IP as HOST, " + " HOST as PROXY_HOST, " - + " SVR_IP || ':' || TO_CHAR(SVR_PORT) AS SVR_IP, " + + " SVR_IP || ':' || TO_CHAR(SQL_PORT) AS SVR_IP, " + " TIME as EXECUTE_TIME, " + " CASE " + " WHEN TRANS_STATE IS NULL OR TRANS_STATE IN ('', 'IDLE', 'IN_TERMINATE', 'ABORTED', " From a53757c91c31d5c519bcb05b060e8681d14863f2 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Thu, 16 Jan 2025 10:43:41 +0800 Subject: [PATCH 114/118] build: upgrade db-browser from 1.2.1 to 1.2.2 #4172 --- libs/db-browser/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/db-browser/pom.xml b/libs/db-browser/pom.xml index 8abba2905d..a571e450d3 100644 --- a/libs/db-browser/pom.xml +++ b/libs/db-browser/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.oceanbase db-browser - 1.2.1 + 1.2.2 db-browser https://github.com/oceanbase/odc/tree/main/libs/db-browser diff --git a/pom.xml b/pom.xml index 99c83c5f6e..f7d81813cd 100644 --- a/pom.xml +++ b/pom.xml @@ -93,7 +93,7 @@ 4.20.19.ALL 4.10.0 2.10.0 - 1.2.1 + 1.2.2 1.4.0 3.10.0 1.64 From dbdabc2795f6091eb86296aaa0b7e678e64a4a9a Mon Sep 17 00:00:00 2001 From: LuckyLeo Date: Thu, 16 Jan 2025 15:04:19 +0800 Subject: [PATCH 115/118] fix(datatransfer): invalid batchSize when importing external csv file * fix invalid batchSize * add default batch size --- .../datatransfer/factory/LoadParameterFactory.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/datatransfer/factory/LoadParameterFactory.java b/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/datatransfer/factory/LoadParameterFactory.java index 2dd52d6f54..65ad35dd50 100644 --- a/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/datatransfer/factory/LoadParameterFactory.java +++ b/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/datatransfer/factory/LoadParameterFactory.java @@ -29,6 +29,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.Validate; +import com.google.common.base.MoreObjects; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.plugin.task.api.datatransfer.model.CsvColumnMapping; import com.oceanbase.odc.plugin.task.api.datatransfer.model.DataTransferConfig; @@ -52,6 +53,8 @@ */ @Slf4j public class LoadParameterFactory extends BaseParameterFactory { + private static final Integer DEFAULT_INSERT_BATCH_SIZE = 100; + public LoadParameterFactory(DataTransferConfig config, File workingDir, File logDir) { super(config, workingDir, logDir); } @@ -83,9 +86,8 @@ protected LoadParameter doGenerate(File workingDir) throws IOException { } if (transferConfig.isTransferData()) { parameter.setTruncatable(transferConfig.isTruncateTableBeforeImport()); - if (transferConfig.getBatchCommitNum() != null) { - parameter.setBatchSize(transferConfig.getBatchCommitNum()); - } + parameter.setBatchSize( + MoreObjects.firstNonNull(transferConfig.getBatchCommitNum(), DEFAULT_INSERT_BATCH_SIZE)); if (transferConfig.getSkippedDataType() != null) { parameter.getExcludeDataTypes().addAll(transferConfig.getSkippedDataType()); } @@ -95,6 +97,11 @@ protected LoadParameter doGenerate(File workingDir) throws IOException { parameter.setTruncatable(transferConfig.isTruncateTableBeforeImport()); setCsvMappings(parameter, transferConfig); setWhiteListForExternalCsv(parameter, transferConfig, workingDir); + parameter.setBatchSize( + MoreObjects.firstNonNull(transferConfig.getBatchCommitNum(), DEFAULT_INSERT_BATCH_SIZE)); + if (transferConfig.getSkippedDataType() != null) { + parameter.getExcludeDataTypes().addAll(transferConfig.getSkippedDataType()); + } } else if (transferConfig.getDataTransferFormat() == DataTransferFormat.SQL) { parameter.setReplaceObjectIfExists(true); } From 792ca382320953d98d636fcf6caa631bda5ed5a7 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Thu, 16 Jan 2025 16:58:20 +0800 Subject: [PATCH 116/118] fix(session): DBSession does not involve svrIp when OB version less than 4.0 #4174 * fix list sessions * fix * fix * fix ut * response to comments * rollback comments * refactor ut --- .../tools/dbbrowser/model/DBMySQLProcess.java | 13 ++++++++++ .../stats/mysql/OBMySQLStatsAccessor.java | 24 +++++++++++++++++++ .../stats/OBMySQLStatsAccessorTest.java | 5 ++-- ...OracleNoLessThan2270StatsAccessorTest.java | 1 + 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBMySQLProcess.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBMySQLProcess.java index 82d0d92b5f..6bcf741cfc 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBMySQLProcess.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBMySQLProcess.java @@ -15,6 +15,8 @@ */ package com.oceanbase.tools.dbbrowser.model; +import com.oceanbase.tools.dbbrowser.util.StringUtils; + import lombok.Data; /** @@ -55,6 +57,16 @@ public class DBMySQLProcess { */ private String host; + /** + * observer ip, refer to the column `Ip` in the result set of `SHOW FULL PROCESSLIST` + */ + private String ip; + + /** + * observer SQL port, refer to the column `Port` in the result set of `SHOW FULL PROCESSLIST` + */ + private String port; + public DBSession toDBSession() { DBSession session = new DBSession(); session.setId(id); @@ -66,6 +78,7 @@ public DBSession toDBSession() { session.setHost(host); session.setProxyHost(host); session.setExecuteTime(Integer.parseInt(time)); + session.setSvrIp(StringUtils.join(ip, ":", port)); return session; } } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLStatsAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLStatsAccessor.java index 4483190185..ec105f1fe4 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLStatsAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLStatsAccessor.java @@ -15,8 +15,15 @@ */ package com.oceanbase.tools.dbbrowser.stats.mysql; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.springframework.jdbc.core.JdbcOperations; +import com.oceanbase.tools.dbbrowser.model.DBSession; +import com.oceanbase.tools.dbbrowser.util.StringUtils; + import lombok.NonNull; /** @@ -28,8 +35,25 @@ */ public class OBMySQLStatsAccessor extends MySQLNoLessThan5700StatsAccessor { + private static final String LIST_SESSIONS_BY_SHOW_PROCESSLIST = "SHOW FULL PROCESSLIST"; + public OBMySQLStatsAccessor(@NonNull JdbcOperations jdbcOperations) { super(jdbcOperations); } + @Override + public List listAllSessions() { + List sessions = super.listAllSessions(); + Map sessionId2SvrIp = new HashMap<>(); + jdbcOperations.query(LIST_SESSIONS_BY_SHOW_PROCESSLIST, rs -> { + if (rs.getMetaData().getColumnCount() == 11) { + String id = rs.getString("Id"); + String svrIp = StringUtils.join(rs.getString("Ip"), ":", rs.getString("Port")); + sessionId2SvrIp.put(id, svrIp); + } + }); + sessions.forEach(session -> session.setSvrIp(sessionId2SvrIp.get(session.getId()))); + return sessions; + } + } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBMySQLStatsAccessorTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBMySQLStatsAccessorTest.java index 3a52ee29f8..1ea11da4fd 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBMySQLStatsAccessorTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBMySQLStatsAccessorTest.java @@ -45,8 +45,9 @@ public void currentSession() { @Test public void listAllSessions() { DBStatsAccessor accessor = new OBMySQLStatsAccessor(new JdbcTemplate(getOBMySQLDataSource())); - List session = accessor.listAllSessions(); - Assert.assertTrue(session.size() > 0); + List sessions = accessor.listAllSessions(); + Assert.assertTrue(sessions.size() > 0); + sessions.stream().forEach(s -> Assert.assertNotNull(s.getSvrIp())); } } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBOracleNoLessThan2270StatsAccessorTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBOracleNoLessThan2270StatsAccessorTest.java index 82feec931a..740110649e 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBOracleNoLessThan2270StatsAccessorTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBOracleNoLessThan2270StatsAccessorTest.java @@ -50,5 +50,6 @@ public void listAllSessions_Succeed() { DBStatsAccessor accessor = new OBOracleNoLessThan2270StatsAccessor(new JdbcTemplate(getOBOracleDataSource())); List sessions = accessor.listAllSessions(); Assert.assertTrue(sessions.size() > 0); + sessions.stream().forEach(s -> Assert.assertNotNull(s.getSvrIp())); } } From 21515069a7788b0d9c0b2275f8e9d0e65bcc5a88 Mon Sep 17 00:00:00 2001 From: Ang <43255684+ungreat@users.noreply.github.com> Date: Fri, 17 Jan 2025 11:37:55 +0800 Subject: [PATCH 117/118] fix(session): kill console query may failed cause session occupied #4175 * new connect session * new connect session * skip permission * refactor --------- Co-authored-by: MarkPotato777 --- .../db/session/DefaultDBSessionManage.java | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index 7d0910c115..857fa4aba8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -84,9 +84,6 @@ public class DefaultDBSessionManage implements DBSessionManageFacade { private static final String GLOBAL_CLIENT_SESSION_OB_PROXY_VERSION_NUMBER = "4.2.3"; private static final String GLOBAL_CLIENT_SESSION_OB_VERSION_NUMBER = "4.2.5"; private static final String ORACLE_MODEL_KILL_SESSION_WITH_BLOCK_OB_VERSION_NUMBER = "4.2.1.0"; - private static final byte GLOBAL_CLIENT_SESSION_PROXY_ID_MIN = 0; - private static final short GLOBAL_CLIENT_SESSION_PROXY_ID_MAX = 8191; - private static final byte GLOBAL_CLIENT_SESSION_ID_VERSION = 2; @Autowired private DBSessionService dbSessionService; @@ -142,10 +139,20 @@ public boolean killConsoleQuery(ConnectionSession session) { Verify.notNull(conn, "ConnectionConfig"); SessionExtensionPoint sessionExtension = ConnectionPluginUtil.getSessionExtension(conn.getDialectType()); - Map connectionId2KillSql = sessionExtension.getKillQuerySqls(SetUtils.hashSet(connectionId)); - List results = doKill(session, connectionId2KillSql); - Verify.singleton(results, "killResults"); - return results.get(0).isKilled(); + DefaultConnectSessionFactory factory = new DefaultConnectSessionFactory(conn); + ConnectionSession copiedSession = null; + try { + copiedSession = factory.generateSession(); + Map connectionId2KillSql = sessionExtension.getKillQuerySqls( + SetUtils.hashSet(connectionId)); + List results = doKill(copiedSession, connectionId2KillSql); + Verify.singleton(results, "killResults"); + return results.get(0).isKilled(); + } finally { + if (copiedSession != null) { + copiedSession.expire(); + } + } } @Override @@ -199,6 +206,7 @@ private List doKill(ConnectionSession session, Map c .collect(Collectors.toList()); } + // Will reuse the ConnectionSession to get the session list private List getSessionList(ConnectionSession connectionSession, Predicate filter) { return DBStatsAccessors.create(connectionSession) .listAllSessions() From 138cd57683ef5d6a177ceace2079e3e11aa446f3 Mon Sep 17 00:00:00 2001 From: pynzzZ Date: Fri, 17 Jan 2025 11:52:08 +0800 Subject: [PATCH 118/118] fix(session): add retry logic when get session list #4176 * add retry * change timeout to 2000L --- .../db/session/DefaultDBSessionManage.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index 857fa4aba8..3cd1bd16b6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -178,11 +178,22 @@ public void killAllSessions(ConnectionSession connectionSession, Predicate checkedList = getSessionList(connectionSession, filter); - if (CollectionUtils.isNotEmpty(checkedList)) { - throw new IllegalStateException("kill session failed, has session reserved"); + long startTimeMs = System.currentTimeMillis(); + long endTimeMs = startTimeMs + 2000L; + while (System.currentTimeMillis() < endTimeMs) { + List checkedList = getSessionList(connectionSession, filter); + if (CollectionUtils.isEmpty(getSessionList(connectionSession, filter))) { + log.info("All sessions killed, elapsed time: {}ms", System.currentTimeMillis() - startTimeMs); + return; + } + log.info("Has sessions reserved:{}", checkedList); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + log.warn("Thread sleep interrupted", e); + } } + throw new IllegalStateException("kill session failed, has session reserved"); } private List doKill(ConnectionSession session, Map connectionId2KillSqls) {