diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 61b492976..cca88c92a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,6 +62,7 @@ jobs: java-version: "17" distribution: "temurin" java-package: "jre" + architecture: "x64" # Install jre MacOS arm64 - name: Install Jre MacOS arm64 @@ -86,7 +87,7 @@ jobs: - name: Enable tls1 if: ${{ runner.os == 'Windows' }} run: | - # sed -i '' "s/\(^jdk.tls.disabledAlgorithms=\)\(.*\)\( TLSv1, TLSv1.1,\)\(.*\)/\1\2\4/" "${{ env.JAVA_HOME }}/conf/security/java.security" + # sed -i '' "s/\(^jdk.tls.disabledAlgorithms=\)\(.*\)\( TLSv1, TLSv1.1,\)\(.*\)/\1\2\4/" "${{ env.JAVA_HOME }}\conf\security\java.security" $filePath = "${{ env.JAVA_HOME }}\conf\security\java.security" $content = Get-Content $filePath -Raw $updatedContent = $content -replace '^(jdk.tls.disabledAlgorithms=)(.*)( TLSv1, TLSv1.1,)(.*)', '$1$2$4' @@ -210,7 +211,7 @@ jobs: run: | xcrun notarytool store-credentials "Chat2DB" --apple-id "${{secrets.MAC_APPLE_ID}}" --password "${{secrets.MAC_APPLE_PASSWORD}}" --team-id "${{secrets.MAC_TEAM_ID}}" xcrun notarytool submit chat2db-client/release/Chat2DB-${{ steps.chat2db_version.outputs.substring }}.dmg --keychain-profile "Chat2DB" - + # macos arm64 - name: Build/release Electron app for MacOS arm64 if: ${{ runner.os == 'macOS' && matrix.arch == 'arm64' }} @@ -346,4 +347,4 @@ jobs: { "title": "Linux-test-打包完成通知", "text": "# Linux-test-打包完成通知 \n ![bang](https://oss.sqlgpt.cn/static/happy100.jpg) \n ### 任务id：[${{ github.run_id }}](https://github.com/chat2db/Chat2DB/actions/runs/${{ github.run_id }}) \n ### Linux下载地址：[https://oss.sqlgpt.cn/release/${{ steps.chat2db_version.outputs.substring }}/Chat2DB-${{ steps.chat2db_version.outputs.substring }}.AppImage](https://oss.sqlgpt.cn/release/${{ steps.chat2db_version.outputs.substring }}/Chat2DB-${{ steps.chat2db_version.outputs.substring }}.AppImage)" - } + } \ No newline at end of file diff --git a/chat2db-client/src/components/ConnectionEdit/config/dataSource.ts b/chat2db-client/src/components/ConnectionEdit/config/dataSource.ts index c95ca70ef..28f51acd7 100644 --- a/chat2db-client/src/components/ConnectionEdit/config/dataSource.ts +++ b/chat2db-client/src/components/ConnectionEdit/config/dataSource.ts @@ -2104,4 +2104,43 @@ export const dataSourceFormConfigs: IConnectionConfig[] = [ }, ssh: sshConfig, }, + // DUCKDB + { + type: DatabaseTypeCode.DUCKDB, + baseInfo: { + items: [ + { + defaultValue: '@localhost', + inputType: InputType.INPUT, + labelNameCN: '名称', + labelNameEN: 'Name', + name: 'alias', + required: true, + }, + envItem, + { + defaultValue: 'localhost', + inputType: InputType.INPUT, + labelNameCN: '路径', + labelNameEN: 'Host', + name: 'host', + required: true, + styles: { + width: '70%', + }, + }, + { + defaultValue: 'jdbc:duckdb:{filePath}', + inputType: InputType.INPUT, + labelNameCN: 'URL', + labelNameEN: 'URL', + name: 'url', + required: true, + }, + ], + pattern: /jdbc:duckdb:\/\/(\w+)/, + template: 'jdbc:duckdb://{host}', + }, + ssh: sshConfig, + }, ]; diff --git a/chat2db-client/src/components/ConnectionEdit/config/enum.ts b/chat2db-client/src/components/ConnectionEdit/config/enum.ts index 6c0e425cf..e66e37e93 100644 --- a/chat2db-client/src/components/ConnectionEdit/config/enum.ts +++ b/chat2db-client/src/components/ConnectionEdit/config/enum.ts @@ -2,6 +2,7 @@ export enum InputType { INPUT = 'input', PASSWORD = 'password', SELECT = 'select', + FILE = 'file', } export enum AuthenticationType { diff --git a/chat2db-client/src/constants/common.ts b/chat2db-client/src/constants/common.ts index 44824008d..8ea26bcbf 100644 --- a/chat2db-client/src/constants/common.ts +++ b/chat2db-client/src/constants/common.ts @@ -16,6 +16,7 @@ export enum DatabaseTypeCode { HIVE = 'HIVE', KINGBASE = 'KINGBASE', TIMEPLUS = 'TIMEPLUS', + DUCKDB = 'DUCKDB', } export enum ConsoleStatus { diff --git a/chat2db-client/src/constants/database.ts b/chat2db-client/src/constants/database.ts index e586cb883..7f174ed5e 100644 --- a/chat2db-client/src/constants/database.ts +++ b/chat2db-client/src/constants/database.ts @@ -125,6 +125,13 @@ export const databaseMap: { // port: 8123, icon: '\ue8f4', }, + [DatabaseTypeCode.DUCKDB]: { + name: 'DuckDB', + img: moreDBLogo, + code: DatabaseTypeCode.DUCKDB, + // port: 8123, + icon: '\ue8f4', + }, // [DatabaseTypeCode.REDIS]: { // name: 'Redis', // img: moreDBLogo, diff --git a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/builder/ClickHouseSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/builder/ClickHouseSqlBuilder.java index 9f313c27c..0eef415af 100644 --- a/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/builder/ClickHouseSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-clickhouse/src/main/java/ai/chat2db/plugin/clickhouse/builder/ClickHouseSqlBuilder.java @@ -10,6 +10,8 @@ import ai.chat2db.spi.model.TableIndex; import org.apache.commons.lang3.StringUtils; +import java.util.List; + public class ClickHouseSqlBuilder extends DefaultSqlBuilder { @Override diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/pom.xml b/chat2db-server/chat2db-plugins/chat2db-duckdb/pom.xml new file mode 100644 index 000000000..7eddf5958 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/pom.xml @@ -0,0 +1,35 @@ + + + + + ai.chat2db + chat2db-plugins + ${revision} + ../pom.xml + + 4.0.0 + chat2db-duckdb + + + + ai.chat2db + chat2db-spi + + + + + + src/main/java + + + **/*.json + + + + src/main/resources + + + + \ No newline at end of file diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBCommandExecutor.java b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBCommandExecutor.java new file mode 100644 index 000000000..faf5a0bd7 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBCommandExecutor.java @@ -0,0 +1,17 @@ +package ai.chat2db.plugin.duckdb; + +import ai.chat2db.spi.model.Command; +import ai.chat2db.spi.model.ExecuteResult; +import ai.chat2db.spi.sql.SQLExecutor; + +import java.util.List; + +public class DuckDBCommandExecutor extends SQLExecutor { + + @Override + public List executeSelectTable(Command command) { + String sql = "select * from " +command.getSchemaName() + "." + command.getTableName(); + command.setScript(sql); + return execute(command); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBManage.java b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBManage.java new file mode 100644 index 000000000..b4c117534 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBManage.java @@ -0,0 +1,253 @@ +package ai.chat2db.plugin.duckdb; + +import ai.chat2db.spi.DBManage; +import ai.chat2db.spi.jdbc.DefaultDBManage; +import ai.chat2db.spi.model.AsyncContext; +import ai.chat2db.spi.model.Function; +import ai.chat2db.spi.model.Procedure; +import ai.chat2db.spi.sql.SQLExecutor; +import cn.hutool.core.date.DateUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.StringUtils; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Date; + +import static cn.hutool.core.date.DatePattern.NORM_DATETIME_PATTERN; + +@Slf4j +public class DuckDBManage extends DefaultDBManage implements DBManage { + + private static String PROCEDURE_SQL = "SELECT COUNT(*) FROM INFORMATION_SCHEMA.ROUTINES " + + "WHERE ROUTINE_SCHEMA = '%s' AND ROUTINE_NAME = '%s' AND ROUTINE_TYPE = 'PROCEDURE'"; + + @Override + public void exportDatabase(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + asyncContext.write(String.format(EXPORT_TITLE, DateUtil.format(new Date(), NORM_DATETIME_PATTERN))); + exportTables(connection, databaseName, schemaName, asyncContext); + asyncContext.setProgress(50); + exportViews(connection, databaseName, asyncContext); + asyncContext.setProgress(60); + exportProcedures(connection, asyncContext); + asyncContext.setProgress(70); + exportTriggers(connection, asyncContext); + asyncContext.setProgress(90); + exportFunctions(connection, databaseName, asyncContext); + asyncContext.finish(); + } + + private void exportFunctions(Connection connection, String databaseName, AsyncContext asyncContext) throws SQLException { + try (ResultSet resultSet = connection.getMetaData().getFunctions(databaseName, null, null)) { + while (resultSet.next()) { + exportFunction(connection, resultSet.getString("FUNCTION_NAME"), asyncContext); + } + + } + } + + private void exportFunction(Connection connection, String functionName, AsyncContext asyncContext) throws SQLException { + String sql = String.format("SHOW CREATE FUNCTION %s;", functionName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + if (resultSet.next()) { + asyncContext.write(String.format(FUNCTION_TITLE, functionName)); + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("DROP FUNCTION IF EXISTS ").append(functionName).append(";").append("\n"); + + sqlBuilder.append("delimiter ;;").append("\n").append(resultSet.getString("Create Function")).append(";;") + .append("\n").append("delimiter ;").append("\n\n"); + asyncContext.write(sqlBuilder.toString()); + } + } + } + + private void exportTables(Connection connection, String databaseName, String schemaName, AsyncContext asyncContext) throws SQLException { + asyncContext.write("SET FOREIGN_KEY_CHECKS=0;"); + try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, null, null, new String[]{"TABLE", "SYSTEM TABLE"})) { + while (resultSet.next()) { + String tableName = resultSet.getString("TABLE_NAME"); + exportTable(connection, databaseName, schemaName, tableName, asyncContext); + } + } + asyncContext.write("SET FOREIGN_KEY_CHECKS=1;"); + } + + + public void exportTable(Connection connection, String databaseName, String schemaName, String tableName, AsyncContext asyncContext) throws SQLException { + String sql = String.format("show create table %s ", tableName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + if (resultSet.next()) { + StringBuilder sqlBuilder = new StringBuilder(); + asyncContext.write(String.format(TABLE_TITLE, tableName)); + sqlBuilder.append("DROP TABLE IF EXISTS ").append(format(tableName)).append(";").append("\n") + .append(resultSet.getString("Create Table")).append(";").append("\n"); + asyncContext.write(sqlBuilder.toString()); + if (asyncContext.isContainsData()) { + exportTableData(connection, databaseName, schemaName, tableName, asyncContext); + } + } + } catch (Exception e) { + log.error("export table error", e); + asyncContext.error(String.format("export table %s error:%s", tableName, e.getMessage())); + } + } + + + private void exportViews(Connection connection, String databaseName, AsyncContext asyncContext) throws SQLException { + try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, null, null, new String[]{"VIEW"})) { + while (resultSet.next()) { + exportView(connection, resultSet.getString("TABLE_NAME"), asyncContext); + } + } + } + + private void exportView(Connection connection, String viewName, AsyncContext asyncContext) throws SQLException { + String sql = String.format("show create view %s ", viewName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + if (resultSet.next()) { + asyncContext.write(String.format(VIEW_TITLE, viewName)); + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("DROP VIEW IF EXISTS ").append(format(viewName)).append(";").append("\n") + .append(resultSet.getString("Create View")).append(";").append("\n\n"); + asyncContext.write(sqlBuilder.toString()); + } + } + } + + private void exportProcedures(Connection connection, AsyncContext asyncContext) throws SQLException { + String sql = "SHOW PROCEDURE STATUS WHERE Db = DATABASE()"; + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + while (resultSet.next()) { + exportProcedure(connection, resultSet.getString("Name"), asyncContext); + } + } + } + + private void exportProcedure(Connection connection, String procedureName, AsyncContext asyncContext) throws SQLException { + String sql = String.format("show create procedure %s ", procedureName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + if (resultSet.next()) { + asyncContext.write(String.format(PROCEDURE_TITLE, procedureName)); + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("DROP PROCEDURE IF EXISTS ").append(format(procedureName)).append(";").append("\n") + .append("delimiter ;;").append("\n").append(resultSet.getString("Create Procedure")).append(";;") + .append("\n").append("delimiter ;").append("\n\n"); + asyncContext.write(sqlBuilder.toString()); + } + } + } + + private void exportTriggers(Connection connection, AsyncContext asyncContext) throws SQLException { + String sql = "SHOW TRIGGERS"; + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + while (resultSet.next()) { + String triggerName = resultSet.getString("Trigger"); + exportTrigger(connection, triggerName, asyncContext); + } + } + } + + private void exportTrigger(Connection connection, String triggerName, AsyncContext asyncContext) throws SQLException { + String sql = String.format("show create trigger %s ", triggerName); + try (ResultSet resultSet = connection.createStatement().executeQuery(sql)) { + if (resultSet.next()) { + asyncContext.write(String.format(TRIGGER_TITLE, triggerName)); + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("DROP TRIGGER IF EXISTS ").append(format(triggerName)).append(";").append("\n") + .append("delimiter ;;").append("\n").append(resultSet.getString("SQL Original Statement")).append(";;") + .append("\n").append("delimiter ;").append("\n\n"); + asyncContext.write(sqlBuilder.toString()); + } + } + } + + @Override + public void updateProcedure(Connection connection, String databaseName, String schemaName, Procedure procedure) throws SQLException { + try { + connection.setAutoCommit(false); + String procedureBody = procedure.getProcedureBody(); + if (procedureBody == null || !procedureBody.trim().toUpperCase().startsWith("CREATE")) { + throw new IllegalArgumentException("No CREATE statement found."); + } + + String procedureNewName = getSchemaOrProcedureName(procedureBody, databaseName, procedure); + if (!procedureNewName.equals(procedure.getProcedureName())) { + procedureBody = procedureBody.replace(procedure.getProcedureName(), procedureNewName); + } + String checkProcedureSQL = String.format(PROCEDURE_SQL, databaseName, procedure.getProcedureName()); + SQLExecutor.getInstance().execute(connection, checkProcedureSQL, resultSet -> { + try { + if (resultSet.next()) { + int count = resultSet.getInt(1); + if (count >= 1) { + throw new Exception("Procedure already exists"); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + }); + SQLExecutor.getInstance().execute(connection, procedureBody, resultSet -> {}); + } catch (Exception e) { + connection.rollback(); + throw new RuntimeException(e); + } finally { + connection.setAutoCommit(true); + } + + } + + @Override + public void connectDatabase(Connection connection, String database) { + if (StringUtils.isEmpty(database)) { + return; + } + try { + SQLExecutor.getInstance().execute(connection, "use " + database + ";"); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + + @Override + public void dropTable(Connection connection, String databaseName, String schemaName, String tableName) { + String sql = "DROP TABLE " + format(tableName); + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } + + @Override + public void deleteProcedure(Connection connection, String databaseName, String schemaName, Procedure procedure) { + String procedureNewName = getSchemaOrProcedureName(procedure.getProcedureBody(), databaseName, procedure); + String sql = "DROP PROCEDURE " + procedureNewName; + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } + + @Override + public void deleteFunction(Connection connection, String databaseName, String schemaName, Function function) { + String functionNewName = getSchemaOrFunctionName(function.getFunctionBody(), databaseName, function); + String sql = "DROP FUNCTION " + functionNewName; + SQLExecutor.getInstance().execute(connection, sql, resultSet -> null); + } + + private static String getSchemaOrProcedureName(String procedureBody, String schemaName, Procedure procedure) { + if (procedureBody.toLowerCase().contains(schemaName.toLowerCase())) { + return procedure.getProcedureName(); + } else { + return schemaName + "." + procedure.getProcedureName(); + } + } + + private static String getSchemaOrFunctionName(String functionBody, String schemaName, Function function) { + if (functionBody.toLowerCase().contains(schemaName.toLowerCase())) { + return function.getFunctionName(); + } else { + return schemaName + "." + function.getFunctionName(); + } + } + + public static String format(String tableName) { + return "`" + tableName + "`"; + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBMetaData.java b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBMetaData.java new file mode 100644 index 000000000..50ef4bd34 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBMetaData.java @@ -0,0 +1,255 @@ +package ai.chat2db.plugin.duckdb; + +import ai.chat2db.plugin.duckdb.builder.DuckDBSqlBuilder; +import ai.chat2db.plugin.duckdb.type.DuckDBColumnTypeEnum; +import ai.chat2db.plugin.duckdb.type.DuckDBDefaultValueEnum; +import ai.chat2db.plugin.duckdb.type.DuckDBIndexTypeEnum; +import ai.chat2db.spi.CommandExecutor; +import ai.chat2db.spi.MetaData; +import ai.chat2db.spi.SqlBuilder; +import ai.chat2db.spi.jdbc.DefaultMetaService; +import ai.chat2db.spi.model.*; +import ai.chat2db.spi.sql.SQLExecutor; +import jakarta.validation.constraints.NotEmpty; +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.stream.Collectors; + +import static ai.chat2db.spi.util.SortUtils.sortDatabase; + +public class DuckDBMetaData extends DefaultMetaService implements MetaData { + + private List systemDatabases = Arrays.asList("information_schema", "temp", "main", "system"); + + @Override + public List databases(Connection connection) { + List databases = SQLExecutor.getInstance().databases(connection); + return sortDatabase(databases, systemDatabases, connection); + } + + @Override + public CommandExecutor getCommandExecutor() { + return new DuckDBCommandExecutor(); + } + + + private static String TABLES_SQL + = "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE' AND TABLE_CATALOG = '%s' AND TABLE_SCHEMA = '%s'"; + @Override + public List tables(Connection connection, @NotEmpty String databaseName, String schemaName, String tableName) { + String sql = String.format(TABLES_SQL, databaseName, schemaName); + if(StringUtils.isNotBlank(tableName)){ + sql += " AND TABLE_NAME = '" + tableName + "'"; + } + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + List
tables = new ArrayList<>(); + while (resultSet.next()) { + Table table = new Table(); + table.setDatabaseName(databaseName); + table.setSchemaName(schemaName); + table.setName(resultSet.getString("table_name")); + //table.setEngine(resultSet.getString("ENGINE")); + //table.setRows(resultSet.getLong("TABLE_ROWS")); + //table.setDataLength(resultSet.getLong("DATA_LENGTH")); + //table.setCreateTime(resultSet.getString("CREATE_TIME")); + //table.setUpdateTime(resultSet.getString("UPDATE_TIME")); + //table.setCollate(resultSet.getString("TABLE_COLLATION")); + table.setComment(resultSet.getString("TABLE_COMMENT")); + tables.add(table); + } + return tables; + }); + } + + + @Override + public String tableDDL(Connection connection, @NotEmpty String databaseName, String schemaName, + @NotEmpty String tableName) { + String sql = "SELECT sql FROM duckdb_tables() WHERE database_name = " + format(databaseName) + + " AND schema_name = " + format(schemaName) + " AND table_name = " + format(tableName); + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + if (resultSet.next()) { + return resultSet.getString("sql"); + } + return null; + }); + } + + public static String format(String tableName) { + return "'" + tableName + "'"; + } + + private static String SELECT_TABLE_COLUMNS = "SELECT * FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' order by ORDINAL_POSITION"; + + @Override + public List columns(Connection connection, String databaseName, String schemaName, String tableName) { + String sql = String.format(SELECT_TABLE_COLUMNS, schemaName, tableName); + List tableColumns = new ArrayList<>(); + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + while (resultSet.next()) { + TableColumn column = new TableColumn(); + column.setDatabaseName(databaseName); + column.setTableName(tableName); + column.setOldName(resultSet.getString("column_name")); + column.setName(resultSet.getString("column_name")); + //column.setColumnType(resultSet.getString("COLUMN_TYPE")); + column.setColumnType(resultSet.getString("data_type").toUpperCase()); + //column.setDataType(resultSet.getInt("DATA_TYPE")); + column.setDefaultValue(resultSet.getString("column_default")); + //column.setAutoIncrement(resultSet.getString("EXTRA").contains("auto_increment")); + column.setComment(resultSet.getString("COLUMN_COMMENT")); + //column.setPrimaryKey("PRI".equalsIgnoreCase(resultSet.getString("COLUMN_KEY"))); + column.setNullable("YES".equalsIgnoreCase(resultSet.getString("is_nullable")) ? 1 : 0); + column.setOrdinalPosition(resultSet.getInt("ordinal_position")); + column.setDecimalDigits(resultSet.getInt("numeric_precision")); + column.setCharSetName(resultSet.getString("character_set_name")); + column.setCollationName(resultSet.getString("collation_name")); + setColumnSize(column, resultSet.getString("data_type")); + tableColumns.add(column); + } + return tableColumns; + }); + } + + private void setColumnSize(TableColumn column, String columnType) { + try { + if (columnType.contains("(")) { + String size = columnType.substring(columnType.indexOf("(") + 1, columnType.indexOf(")")); + if ("SET".equalsIgnoreCase(column.getColumnType()) || "ENUM".equalsIgnoreCase(column.getColumnType())) { + column.setValue(size); + } else { + if (size.contains(",")) { + String[] sizes = size.split(","); + if (StringUtils.isNotBlank(sizes[0])) { + column.setColumnSize(Integer.parseInt(sizes[0])); + } + if (StringUtils.isNotBlank(sizes[1])) { + column.setDecimalDigits(Integer.parseInt(sizes[1])); + } + } else { + column.setColumnSize(Integer.parseInt(size)); + } + } + } + } catch (Exception e) { + } + } + + private static String VIEW_DDL_SQL = "SELECT sql FROM duckdb_views() WHERE database_name = '%s' AND schema_name = '%s' AND view_name = '%s'"; + + @Override + public Table view(Connection connection, String databaseName, String schemaName, String viewName) { + String sql = String.format(VIEW_DDL_SQL, databaseName, schemaName, viewName); + return SQLExecutor.getInstance().execute(connection, sql, resultSet -> { + Table table = new Table(); + table.setDatabaseName(databaseName); + table.setSchemaName(schemaName); + table.setName(viewName); + if (resultSet.next()) { + table.setDdl(resultSet.getString("sql")); + } + return table; + }); + } + + + @Override + public List indexes(Connection connection, String databaseName, String schemaName, String tableName) { + StringBuilder queryBuf = new StringBuilder("SELECT * FROM duckdb_indexes WHERE schema_name = "); + queryBuf.append("'").append(schemaName).append("'"); + queryBuf.append(" and table_name = "); + queryBuf.append("'").append(tableName).append("'"); + return SQLExecutor.getInstance().execute(connection, queryBuf.toString(), resultSet -> { + LinkedHashMap map = new LinkedHashMap(); + while (resultSet.next()) { + String keyName = resultSet.getString("Key_name"); + TableIndex tableIndex = map.get(keyName); + if (tableIndex != null) { + List columnList = tableIndex.getColumnList(); + columnList.add(getTableIndexColumn(resultSet)); + columnList = columnList.stream().sorted(Comparator.comparing(TableIndexColumn::getOrdinalPosition)) + .collect(Collectors.toList()); + tableIndex.setColumnList(columnList); + } else { + TableIndex index = new TableIndex(); + index.setDatabaseName(databaseName); + index.setSchemaName(schemaName); + index.setTableName(tableName); + index.setName(keyName); + index.setUnique(!resultSet.getBoolean("Non_unique")); + index.setType(resultSet.getString("Index_type")); + index.setComment(resultSet.getString("Index_comment")); + List tableIndexColumns = new ArrayList<>(); + tableIndexColumns.add(getTableIndexColumn(resultSet)); + index.setColumnList(tableIndexColumns); + if ("PRIMARY".equalsIgnoreCase(keyName)) { + index.setType(DuckDBIndexTypeEnum.PRIMARY_KEY.getName()); + } else if (index.getUnique()) { + index.setType(DuckDBIndexTypeEnum.UNIQUE.getName()); + } else { + index.setType(DuckDBIndexTypeEnum.NORMAL.getName()); + } + map.put(keyName, index); + } + } + return map.values().stream().collect(Collectors.toList()); + }); + + } + + private TableIndexColumn getTableIndexColumn(ResultSet resultSet) throws SQLException { + TableIndexColumn tableIndexColumn = new TableIndexColumn(); + tableIndexColumn.setColumnName(resultSet.getString("Column_name")); + tableIndexColumn.setOrdinalPosition(resultSet.getShort("Seq_in_index")); + tableIndexColumn.setCollation(resultSet.getString("Collation")); + tableIndexColumn.setCardinality(resultSet.getLong("Cardinality")); + tableIndexColumn.setSubPart(resultSet.getLong("Sub_part")); + String collation = resultSet.getString("Collation"); + if ("a".equalsIgnoreCase(collation)) { + tableIndexColumn.setAscOrDesc("ASC"); + } else if ("d".equalsIgnoreCase(collation)) { + tableIndexColumn.setAscOrDesc("DESC"); + } + return tableIndexColumn; + } + + @Override + public SqlBuilder getSqlBuilder() { + return new DuckDBSqlBuilder(); + } + + @Override + public TableMeta getTableMeta(String databaseName, String schemaName, String tableName) { + return TableMeta.builder() + .columnTypes(DuckDBColumnTypeEnum.getTypes()) + //.collations(MysqlCollationEnum.getCollations()) + .indexTypes(DuckDBIndexTypeEnum.getIndexTypes()) + .defaultValues(DuckDBDefaultValueEnum.getDefaultValues()) + .build(); + } + + @Override + public String getMetaDataName(String... names) { + return Arrays.stream(names).filter(name -> StringUtils.isNotBlank(name)).collect(Collectors.joining(".")); + } + +// @Override +// public ValueHandler getValueHandler() { +// return new MysqlValueHandler(); +// } + + /*@Override + public ValueProcessor getValueProcessor() { + return new DuckDBValueProcessor(); + }*/ + + @Override + public List getSystemDatabases() { + return systemDatabases; + } + +} diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBPlugin.java b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBPlugin.java new file mode 100644 index 000000000..648ecd3aa --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/DuckDBPlugin.java @@ -0,0 +1,25 @@ +package ai.chat2db.plugin.duckdb; + +import ai.chat2db.spi.DBManage; +import ai.chat2db.spi.MetaData; +import ai.chat2db.spi.Plugin; +import ai.chat2db.spi.config.DBConfig; +import ai.chat2db.spi.util.FileUtils; + +public class DuckDBPlugin implements Plugin { + + @Override + public DBConfig getDBConfig() { + return FileUtils.readJsonValue(this.getClass(),"duckDB.json", DBConfig.class); + } + + @Override + public MetaData getMetaData() { + return new DuckDBMetaData(); + } + + @Override + public DBManage getDBManage() { + return new DuckDBManage(); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/builder/DuckDBSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/builder/DuckDBSqlBuilder.java new file mode 100644 index 000000000..6a981ad6c --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/builder/DuckDBSqlBuilder.java @@ -0,0 +1,175 @@ +package ai.chat2db.plugin.duckdb.builder; + +import ai.chat2db.plugin.duckdb.type.DuckDBColumnTypeEnum; +import ai.chat2db.plugin.duckdb.type.DuckDBIndexTypeEnum; +import ai.chat2db.spi.enums.EditStatus; +import ai.chat2db.spi.jdbc.DefaultSqlBuilder; +import ai.chat2db.spi.model.Database; +import ai.chat2db.spi.model.Table; +import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.model.TableIndex; +import ai.chat2db.spi.util.SqlUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + + +public class DuckDBSqlBuilder extends DefaultSqlBuilder { + @Override + public String buildCreateTableSql(Table table) { + StringBuilder script = new StringBuilder(); + script.append("CREATE TABLE "); + if (StringUtils.isNotBlank(table.getSchemaName())) { + script.append(table.getSchemaName()).append("."); + } + script.append(table.getName()).append(" (").append("\n"); + + // append column + for (TableColumn column : table.getColumnList()) { + if (StringUtils.isBlank(column.getName()) || StringUtils.isBlank(column.getColumnType())) { + continue; + } + DuckDBColumnTypeEnum typeEnum = DuckDBColumnTypeEnum.getByType(column.getColumnType()); + if (typeEnum == null) { + continue; + } + script.append("\t").append(typeEnum.buildCreateColumnSql(column)).append(",\n"); + } + + // append primary key and index + for (TableIndex tableIndex : table.getIndexList()) { + if (StringUtils.isBlank(tableIndex.getName()) || StringUtils.isBlank(tableIndex.getType())) { + continue; + } + DuckDBIndexTypeEnum mysqlIndexTypeEnum = DuckDBIndexTypeEnum.getByType(tableIndex.getType()); + if (mysqlIndexTypeEnum == null) { + continue; + } + script.append("\t").append(mysqlIndexTypeEnum.buildCreateIndexScript(tableIndex)).append(",\n"); + } + + script = new StringBuilder(script.substring(0, script.length() - 2)); + script.append("\n);\n"); + + + if (StringUtils.isNotBlank(table.getComment())) { + script.append(" COMMENT ON TABLE ").append(table.getSchemaName()).append(".").append(table.getName()) + .append(" IS '").append(table.getComment()).append("'"); + } + + script.append(";"); + + return script.toString(); + } + + @Override + public String buildModifyTaleSql(Table oldTable, Table newTable) { + StringBuilder tableBuilder = new StringBuilder(); + + if (!StringUtils.equalsIgnoreCase(oldTable.getName(), newTable.getName())) { + tableBuilder.append("ALTER TABLE ").append(oldTable.getSchemaName()).append(".").append(oldTable.getName()) + .append(" RENAME TO ").append("'").append(newTable.getName()).append("'").append(";\n"); + } + + if (!StringUtils.equalsIgnoreCase(oldTable.getComment(), newTable.getComment())) { + tableBuilder.append("COMMENT ON TABLE ").append(oldTable.getSchemaName()).append(".").append(oldTable.getName()) + .append(" IS ").append("'").append(newTable.getComment()).append("'").append(";\n"); + } + + + // append modify column + for (TableColumn tableColumn : newTable.getColumnList()) { + if ((StringUtils.isNotBlank(tableColumn.getEditStatus()) && StringUtils.isNotBlank(tableColumn.getColumnType()) + && StringUtils.isNotBlank(tableColumn.getName()))) { + DuckDBColumnTypeEnum typeEnum = DuckDBColumnTypeEnum.getByType(tableColumn.getColumnType()); + if (typeEnum == null) { + continue; + } + tableBuilder.append("\t").append(typeEnum.buildModifyColumn(tableColumn)).append("\n"); + + } + } + + // append modify index + for (TableIndex tableIndex : newTable.getIndexList()) { + if (StringUtils.isNotBlank(tableIndex.getEditStatus()) && StringUtils.isNotBlank(tableIndex.getType())) { + DuckDBIndexTypeEnum duckDBIndexTypeEnum = DuckDBIndexTypeEnum.getByType(tableIndex.getType()); + if (duckDBIndexTypeEnum == null) { + continue; + } + tableBuilder.append("\t").append(duckDBIndexTypeEnum.buildModifyIndex(tableIndex)).append(";\n"); + } + } + + // append reorder column + // script.append(buildGenerateReorderColumnSql(oldTable, newTable)); + + if (tableBuilder.length() > 2) { + tableBuilder = new StringBuilder(tableBuilder.substring(0, tableBuilder.length() - 2)); + tableBuilder.append(";"); + return tableBuilder.toString(); + } else { + return StringUtils.EMPTY; + } + + } + + @Override + public String pageLimit(String sql, int offset, int pageNo, int pageSize) { + StringBuilder sqlBuilder = new StringBuilder(sql.length() + 14); + sqlBuilder.append(sql); + if (offset == 0) { + sqlBuilder.append("\n LIMIT "); + sqlBuilder.append(pageSize); + } else { + sqlBuilder.append("\n LIMIT "); + sqlBuilder.append(offset); + sqlBuilder.append(","); + sqlBuilder.append(pageSize); + } + return sqlBuilder.toString(); + } + + + @Override + public String buildCreateDatabaseSql(Database database) { + StringBuilder sqlBuilder = new StringBuilder(); + sqlBuilder.append("CREATE DATABASE " + database.getName()); + if (StringUtils.isNotBlank(database.getCharset())) { + sqlBuilder.append(" DEFAULT CHARACTER SET=").append(database.getCharset()); + } + if (StringUtils.isNotBlank(database.getCollation())) { + sqlBuilder.append(" COLLATE=").append(database.getCollation()); + } + return sqlBuilder.toString(); + } + + + @Override + protected void buildTableName(String databaseName, String schemaName, String tableName, StringBuilder script) { + if (StringUtils.isNotBlank(databaseName)) { + script.append(SqlUtils.quoteObjectName(databaseName, "'")).append('.'); + } + if (StringUtils.isNotBlank(schemaName)) { + script.append(SqlUtils.quoteObjectName(schemaName, "'")).append('.'); + } + script.append(SqlUtils.quoteObjectName(tableName, "'")); + } + + /** + * @param columnList + * @param script + */ + @Override + protected void buildColumns(List columnList, StringBuilder script) { + if (CollectionUtils.isNotEmpty(columnList)) { + script.append(" (") + .append(columnList.stream().map(s -> SqlUtils.quoteObjectName(s, "`")).collect(Collectors.joining(","))) + .append(") "); + } + } + +} diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/builder/form.json b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/builder/form.json new file mode 100644 index 000000000..e24020fbe --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/builder/form.json @@ -0,0 +1,45 @@ +{ + "baseInfo": { + "items": [ + { + "defaultValue": "@localhost", + "inputType": "INPUT", + "labelNameCN": "名称", + "labelNameEN": "Name", + "name": "alias", + "required": true, + "width": 100, + }, + { + "defaultValue": "localhost", + "inputType": "INPUT", + "labelNameCN": "主机", + "labelNameEN": "Host", + "name": "host", + "required": true, + "width": 70, + }, + { + "defaultValue": "", + "inputType": "INPUT", + "labelNameCN": "数据库", + "labelNameEN": "Database", + "name": "database", + "required": false, + "width": 100 + }, + { + "defaultValue": "jdbc:duckdb:{file}", + "inputType": "INPUT", + "labelNameCN": "URL", + "labelNameEN": "URL", + "name": "url", + "required": true, + "width": 100 + } + ], + "pattern": "/jdbc:duckdb:\/\/(\\w+)", + "template": "jdbc:duckdb://{host}" + }, + "type":"DuckDB" +} \ No newline at end of file diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/duckDB.json b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/duckDB.json new file mode 100644 index 000000000..0c4dec309 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/duckDB.json @@ -0,0 +1,18 @@ +{ + "dbType": "DUCKDB", + "supportDatabase": true, + "supportSchema": true, + "driverConfigList": [ + { + "url": "jdbc:duckdb://", + "defaultDriver": true, + "custom": false, + "downloadJdbcDriverUrls": [ + "https://cdn.chat2db-ai.com/lib/duckdb_jdbc-1.1.3.jar" + ], + "jdbcDriver": "duckdb_jdbc-1.1.3.jar", + "jdbcDriverClass": "org.duckdb.DuckDBDriver" + } + ], + "name": "DuckDB" +} diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/type/DuckDBColumnTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/type/DuckDBColumnTypeEnum.java new file mode 100644 index 000000000..9b4374284 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/type/DuckDBColumnTypeEnum.java @@ -0,0 +1,413 @@ +package ai.chat2db.plugin.duckdb.type; + +import ai.chat2db.spi.ColumnBuilder; +import ai.chat2db.spi.enums.EditStatus; +import ai.chat2db.spi.model.ColumnType; +import ai.chat2db.spi.model.TableColumn; +import ai.chat2db.spi.util.SqlUtils; +import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public enum DuckDBColumnTypeEnum implements ColumnBuilder { + + JSON("JSON", false, false, true, false, false, false, true, true, false, false), + + BIGINT("BIGINT", false, false, true, false, false, false, true, true, false, false), + + BINARY("BINARY", false, false, true, false, false, false, true, true, false, false), + + BIT("BIT", false, false, true, false, false, false, true, true, false, false), + + BLOB("BLOB", false, false, true, false, false, false, true, true, false, false), + + BOOL("BOOL", false, false, true, false, false, false, true, true, false, false), + + BOOLEAN("BOOLEAN", false, false, true, false, false, false, true, true, false, false), + + BPCHAR("BPCHAR", false, false, true, false, false, false, true, true, false, false), + + BYTEA("BYTEA", false, false, true, false, false, false, true, true, false, false), + + CHAR("CHAR", true, false, true, false, false, false, true, true, false, false), + + DATE("DATE", false, false, true, false, false, false, true, true, false, false), + + DATETIME("DATETIME", false, false, true, false, false, false, true, true, false, false), + + DEC("DEC", false, false, true, false, false, false, true, true, false, false), + + DECIMAL("DECIMAL", true, true, true, false, false, false, true, true, false, false), + + DOUBLE("DOUBLE", false, false, true, false, false, false, true, true, false, false), + + ENUM("ENUM", false, false, true, false, false, false, true, true, false, false), + + FLOAT("FLOAT", true, false, true, false, false, false, true, true, false, false), + + FLOAT4("FLOAT4", true, false, true, false, false, false, true, true, false, false), + + FLOAT8("FLOAT8", true, false, true, false, false, false, true, true, false, false), + + GUID("GUID", false, false, true, false, false, false, true, true, false, false), + + HUGEINT("HUGEINT", false, false, true, true, false, false, true, true, false, false), + + INT("INT", false, false, true, false, false, false, true, true, false, false), + INT1("INT1", false, false, true, false, false, false, true, true, false, false), + INT128("INT128", false, false, true, false, false, false, true, true, false, false), + INT16("INT16", false, false, true, false, false, false, true, true, false, false), + INT2("INT2", false, false, true, false, false, false, true, true, false, false), + INT32("INT32", false, false, true, false, false, false, true, true, false, false), + INT4("INT4", false, false, true, false, false, false, true, true, false, false), + INT64("INT64", false, false, true, false, false, false, true, true, false, false), + INT8("INT8", false, false, true, false, false, false, true, true, false, false), + + INTEGER("INTEGER", false, false, true, false, false, false, true, true, false, false), + + INTEGRAL("INTEGRAL", false, false, true, false, false, false, true, true, false, false), + INTERVAL("INTERVAL", false, false, true, false, false, false, true, true, false, false), + + LIST("LIST", false, false, true, false, false, false, true, true, false, false), + + LOGICAL("LOGICAL", false, false, true, false, false, false, true, true, false, false), + + LONG("LONG", false, false, true, false, false, false, true, true, false, false), + + MAP("MAP", false, false, true, false, false, false, true, true, false, false), + + NULL("NULL", false, false, true, false, false, false, true, true, false, false), + + NUMERIC("NUMERIC", false, false, true, false, false, false, true, true, false, false), + + NVARCHAR("NVARCHAR", true, false, true, false, false, false, true, true, false, false), + + OID("OID", false, false, true, false, false, false, true, true, false, false), + + REAL("REAL", false, false, true, false, false, false, true, true, false, false), + + ROW("ROW", false, false, true, false, false, false, true, true, false, false), + + SHORT("SHORT", false, false, true, false, false, false, true, true, false, false), + + SIGNED("SIGNED", false, false, true, false, false, false, true, true, false, false), + + SMALLINT("SMALLINT", false, false, true, false, false, false, true, true, false, false), + + STRING("STRING", false, false, true, false, false, false, true, true, false, false), + + STRUCT("STRUCT", false, false, true, false, false, false, true, true, false, false), + + TEXT("TEXT", false, false, true, false, false, false, true, true, false, false), + + TIME("TIME", false, false, true, false, false, false, true, true, false, false), + + TIMESTAMP("TIMESTAMP", false, false, true, false, false, false, true, true, false, false), + + TIMSTAMP_MS("TIMSTAMP_MS", false, false, true, false, false, false, true, true, false, false), + + TIMSTAMP_NS("TIMSTAMP_NS", false, false, true, false, false, false, true, true, false, false), + TIMSTAMP_S("TIMSTAMP_S", false, false, true, false, false, false, true, true, false, false), + TIMSTAMP_US("TIMSTAMP_US", false, false, true, false, false, false, true, true, false, false), + + TIMESTAMP_WITH_TIME_ZONE("TIMESTAMP WITH TIME ZONE", false, false, true, false, false, false, true, true, false, false), + + TIME_WITH_TIME_ZONE("TIME WITH TIME ZONE", false, false, true, false, false, false, true, true, false, false), + + + TINYINT("TINYINT", false, false, true, false, false, false, true, true, false, false), + + UBIGINT("UBIGINT", false, false, true, false, false, false, true, true, false, false), + + UHUGEINT("UHUGEINT", false, false, true, false, false, false, true, true, false, false), + + UINT128("UINT128", false, false, true, false, false, false, true, true, false, false), + + UINT16("UINT16", false, false, true, false, false, false, true, true, false, false), + + UINT32("UINT32", false, false, true, false, false, false, true, true, false, false), + + UINT64("UINT64", false, false, true, false, false, false, true, true, false, false), + + UINT8("UINT8", false, false, true, false, false, false, true, true, false, false), + + UINTEGER("UINTEGER", false, false, true, false, false, false, true, true, false, false), + + UNION("UNION", false, false, true, false, false, false, true, true, false, false), + + USMALLINT("USMALLINT", false, false, true, false, false, false, true, true, false, false), + + UTINYINT("UTINYINT", false, false, true, false, false, false, true, true, false, false), + + UUID("UUID", false, false, true, false, false, false, true, true, false, false), + + VARBINARY("VARBINARY", false, false, true, false, false, false, true, true, false, false), + + VARCHAR("VARCHAR", true, false, true, false, false, false, true, true, false, true), + + VARINT("VARINT", false, false, true, false, false, false, true, true, false, false), + + ARRAY("ARRAY", false, false, true, false, false, false, true, true, false, false), + ; + private ColumnType columnType; + + public static DuckDBColumnTypeEnum getByType(String dataType) { + String type = SqlUtils.removeDigits(dataType.toUpperCase()); + return COLUMN_TYPE_MAP.get(type); + } + + private static Map COLUMN_TYPE_MAP = Maps.newHashMap(); + + static { + for (DuckDBColumnTypeEnum value : DuckDBColumnTypeEnum.values()) { + COLUMN_TYPE_MAP.put(value.getColumnType().getTypeName(), value); + } + } + + public ColumnType getColumnType() { + return columnType; + } + + + DuckDBColumnTypeEnum(String dataTypeName, boolean supportLength, boolean supportScale, boolean supportNullable, boolean supportAutoIncrement, boolean supportCharset, boolean supportCollation, boolean supportComments, boolean supportDefaultValue, boolean supportExtent, boolean supportUnit) { + this.columnType = new ColumnType(dataTypeName, supportLength, supportScale, supportNullable, supportAutoIncrement, supportCharset, supportCollation, supportComments, supportDefaultValue, supportExtent, false, supportUnit); + } + + @Override + public String buildCreateColumnSql(TableColumn column) { + DuckDBColumnTypeEnum type = COLUMN_TYPE_MAP.get(column.getColumnType().toUpperCase()); + if (type == null) { + return ""; + } + StringBuilder script = new StringBuilder(); + + script.append(column.getName()).append(" "); + + script.append(buildDataType(column, type)).append(" "); + + script.append(buildDefaultValue(column, type)).append(" "); + + script.append(buildAutoIncrement(column,type)).append(" "); + + script.append(buildCreateNullable(column, type)).append(" "); + + return script.toString(); + } + + private String buildAutoIncrement(TableColumn column, DuckDBColumnTypeEnum type) { + if(!type.getColumnType().isSupportAutoIncrement()){ + return ""; + } + if (column.getAutoIncrement() != null && column.getAutoIncrement() + && column.getSeed() != null && column.getSeed() > 0 && column.getIncrement() != null && column.getIncrement() > 0) { + return "IDENTITY(" + column.getSeed() + "," + column.getIncrement() + ")"; + } + if (column.getAutoIncrement() != null && column.getAutoIncrement()) { + return "IDENTITY(1,1)"; + } + return ""; + } + + private String buildNullable(TableColumn column, DuckDBColumnTypeEnum type) { + if (!type.getColumnType().isSupportNullable()) { + return ""; + } + if (column.getNullable() != null && 1 == column.getNullable()) { + return "DROP NOT NULL"; + } else { + return "SET NOT NULL"; + } + } + + private String buildCreateNullable(TableColumn column, DuckDBColumnTypeEnum type) { + if (!type.getColumnType().isSupportNullable()) { + return ""; + } + if (column.getNullable() != null && 1 == column.getNullable()) { + return ""; + } else { + return "NOT NULL"; + } + } + + private String buildDefaultValue(TableColumn column, DuckDBColumnTypeEnum type) { + if (!type.getColumnType().isSupportDefaultValue() || StringUtils.isEmpty(column.getDefaultValue())) { + return ""; + } + + StringBuilder script = new StringBuilder(); + script.append("ALTER TABLE ").append(column.getSchemaName()).append(".").append(column.getTableName()); + script.append(column.getOldName()).append(" "); + if ("EMPTY_STRING".equalsIgnoreCase(column.getDefaultValue().trim())) { + return script.append("SET DEFAULT '';\n").toString(); + } + + if ("NULL".equalsIgnoreCase(column.getDefaultValue().trim())) { + return script.append("SET DEFAULT NULL;\n").toString(); + } + + return script.append("SET DEFAULT ").append(column.getDefaultValue()).append(";\n").toString(); + } + + private String buildDataType(TableColumn column, DuckDBColumnTypeEnum type) { + String columnType = type.columnType.getTypeName(); + if (Arrays.asList(VARCHAR, STRING, BPCHAR, NVARCHAR, TEXT).contains(type)) { + StringBuilder script = new StringBuilder(); + script.append(columnType); + if (column.getColumnSize() != null && StringUtils.isEmpty(column.getUnit())) { + script.append("(").append(column.getColumnSize()).append(")"); + } else if (column.getColumnSize() != null && !StringUtils.isEmpty(column.getUnit())) { + script.append("(").append(column.getColumnSize()).append(" ").append(column.getUnit()).append(")"); + } + return script.toString(); + } + + if (Arrays.asList(DECIMAL, FLOAT, TIMESTAMP).contains(type)) { + StringBuilder script = new StringBuilder(); + script.append(columnType); + if (column.getColumnSize() != null && column.getDecimalDigits() == null) { + script.append("(").append(column.getColumnSize()).append(")"); + } else if (column.getColumnSize() != null && column.getDecimalDigits() != null) { + script.append("(").append(column.getColumnSize()).append(",").append(column.getDecimalDigits()).append(")"); + } + return script.toString(); + } + + if (Arrays.asList(TIME_WITH_TIME_ZONE, TIMSTAMP_US).contains(type)) { + StringBuilder script = new StringBuilder(); + if (column.getColumnSize() == null) { + script.append(columnType); + } else { + String[] split = columnType.split("TIMESTAMP"); + script.append("TIMESTAMP").append("(").append(column.getColumnSize()).append(")").append(split[1]); + } + return script.toString(); + } + return columnType; + } + + private String buildModifyDataType(TableColumn column, DuckDBColumnTypeEnum type) { + String columnType = type.columnType.getTypeName(); + if (Arrays.asList(VARCHAR, STRING, BPCHAR, NVARCHAR, TEXT).contains(type)) { + StringBuilder script = new StringBuilder(); + script.append(columnType); + if (column.getColumnSize() != null && StringUtils.isEmpty(column.getUnit())) { + script.append("(").append(column.getColumnSize()).append(")"); + } else if (column.getColumnSize() != null && !StringUtils.isEmpty(column.getUnit())) { + script.append("(").append(column.getColumnSize()).append(" ").append(column.getUnit()).append(")"); + } + return script.toString(); + } + + if (Arrays.asList(DECIMAL, FLOAT, TIMESTAMP).contains(type)) { + StringBuilder script = new StringBuilder(); + script.append(columnType); + if (column.getColumnSize() != null && column.getDecimalDigits() == null) { + script.append("(").append(column.getColumnSize()).append(")"); + } else if (column.getColumnSize() != null && column.getDecimalDigits() != null) { + script.append("(").append(column.getColumnSize()).append(",").append(column.getDecimalDigits()).append(")"); + } + return script.toString(); + } + + if (Arrays.asList(TIME_WITH_TIME_ZONE, TIMSTAMP_US).contains(type)) { + StringBuilder script = new StringBuilder(); + if (column.getColumnSize() == null) { + script.append(columnType); + } else { + String[] split = columnType.split("TIMESTAMP"); + script.append("TIMESTAMP").append("(").append(column.getColumnSize()).append(")").append(split[1]); + } + return script.toString(); + } + return columnType; + } + + + + @Override + public String buildModifyColumn(TableColumn tableColumn) { + + if (EditStatus.DELETE.name().equals(tableColumn.getEditStatus())) { + StringBuilder script = new StringBuilder(); + script.append("ALTER TABLE ").append(tableColumn.getSchemaName()).append(".").append(tableColumn.getTableName()); + script.append(" ").append("DROP ").append(tableColumn.getName()).append(";\n"); + return script.toString(); + } + if (EditStatus.ADD.name().equals(tableColumn.getEditStatus())) { + StringBuilder script = new StringBuilder(); + script.append(buildModifyADDColumnSql(tableColumn)).append(";\n"); + return script.toString(); + } + if (EditStatus.MODIFY.name().equals(tableColumn.getEditStatus())) { + StringBuilder script = new StringBuilder(); + + script.append(buildModifyColumnSql(tableColumn, tableColumn.getOldColumn())).append(" \n"); + + if (!StringUtils.equalsIgnoreCase(tableColumn.getOldName(), tableColumn.getName())) { + script.append("ALTER TABLE ").append(tableColumn.getSchemaName()).append(".").append(tableColumn.getTableName()); + script.append(" ").append("RENAME ").append(tableColumn.getOldName()).append(" TO ").append(tableColumn.getName()).append(";\n"); + } + return script.toString(); + + } + return ""; + } + + public String buildModifyColumnSql(TableColumn column, TableColumn oldColumn) { + DuckDBColumnTypeEnum type = COLUMN_TYPE_MAP.get(column.getColumnType().toUpperCase()); + if (type == null) { + return ""; + } + StringBuilder script = new StringBuilder(); + + if (!column.getColumnType().equals(oldColumn.getColumnType())) { + script.append("ALTER TABLE ").append(column.getSchemaName()).append(".").append(column.getTableName()).append(" "); + script.append("ALTER ").append(oldColumn.getName()).append(" SET DATA TYPE ").append(buildModifyDataType(column, type)).append(";\n"); + } + + script.append(buildDefaultValue(column, type)).append(" "); + + if (oldColumn.getNullable() != column.getNullable()) { + script.append("ALTER TABLE ").append(column.getSchemaName()).append(".").append(column.getTableName()).append(" "); + script.append("ALTER COLUMN ").append(column.getName()).append(" ").append(buildNullable(column, type)).append(";\n"); + } + + return script.toString(); + } + + public String buildModifyADDColumnSql(TableColumn column) { + DuckDBColumnTypeEnum type = COLUMN_TYPE_MAP.get(column.getColumnType().toUpperCase()); + if (type == null) { + return ""; + } + StringBuilder script = new StringBuilder(); + script.append("ALTER TABLE ").append(column.getSchemaName()).append(".").append(column.getTableName()); + script.append(" ").append("ADD COLUMN ").append(column.getName()).append(" ").append(buildModifyDataType(column, type)); + if (!type.getColumnType().isSupportDefaultValue() || StringUtils.isEmpty(column.getDefaultValue())) { + return script.append(";\n").toString(); + } else { + if ("EMPTY_STRING".equalsIgnoreCase(column.getDefaultValue().trim())) { + return script.append(" ").append("DEFAULT '';\n").toString(); + } + + if ("NULL".equalsIgnoreCase(column.getDefaultValue().trim())) { + return script.append(" ").append("DEFAULT NULL;\n").toString(); + } + script.append(" ").append("DEFAULT ").append(column.getDefaultValue()).append(";\n"); + } + + + + return script.toString(); + } + + public static List getTypes() { + return Arrays.stream(DuckDBColumnTypeEnum.values()).map(columnTypeEnum -> + columnTypeEnum.getColumnType() + ).toList(); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/type/DuckDBDefaultValueEnum.java b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/type/DuckDBDefaultValueEnum.java new file mode 100644 index 000000000..ee1152776 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/type/DuckDBDefaultValueEnum.java @@ -0,0 +1,27 @@ +package ai.chat2db.plugin.duckdb.type; + +import ai.chat2db.spi.model.DefaultValue; + +import java.util.Arrays; +import java.util.List; + +public enum DuckDBDefaultValueEnum { + EMPTY_STRING("EMPTY_STRING"), + NULL("NULL"), + ; + private DefaultValue defaultValue; + + DuckDBDefaultValueEnum(String defaultValue) { + this.defaultValue = new DefaultValue(defaultValue); + } + + + public DefaultValue getDefaultValue() { + return defaultValue; + } + + public static List getDefaultValues() { + return Arrays.stream(DuckDBDefaultValueEnum.values()).map(DuckDBDefaultValueEnum::getDefaultValue).collect(java.util.stream.Collectors.toList()); + } + +} diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/type/DuckDBIndexTypeEnum.java b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/type/DuckDBIndexTypeEnum.java new file mode 100644 index 000000000..eaea01003 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/java/ai/chat2db/plugin/duckdb/type/DuckDBIndexTypeEnum.java @@ -0,0 +1,143 @@ +package ai.chat2db.plugin.duckdb.type; + +import ai.chat2db.spi.enums.EditStatus; +import ai.chat2db.spi.model.IndexType; +import ai.chat2db.spi.model.TableIndex; +import ai.chat2db.spi.model.TableIndexColumn; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.List; + +public enum DuckDBIndexTypeEnum { + + PRIMARY_KEY("Primary", "PRIMARY KEY"), + + NORMAL("Normal", "INDEX"), + + UNIQUE("Unique", "UNIQUE INDEX"), + + BITMAP("BITMAP", "BITMAP INDEX"); + + + + public IndexType getIndexType() { + return indexType; + } + + public void setIndexType(IndexType indexType) { + this.indexType = indexType; + } + + private IndexType indexType; + + + public String getName() { + return name; + } + + private String name; + + + public String getKeyword() { + return keyword; + } + + private String keyword; + + DuckDBIndexTypeEnum(String name, String keyword) { + this.name = name; + this.keyword = keyword; + this.indexType = new IndexType(name); + } + + + public static DuckDBIndexTypeEnum getByType(String type) { + for (DuckDBIndexTypeEnum value : DuckDBIndexTypeEnum.values()) { + if (value.name.equalsIgnoreCase(type)) { + return value; + } + } + return null; + } + + public String buildIndexScript(TableIndex tableIndex) { + StringBuilder script = new StringBuilder(); + if (PRIMARY_KEY.equals(this)) { + script.append("ALTER TABLE ").append(tableIndex.getSchemaName()).append(".").append(tableIndex.getTableName()).append(" ADD PRIMARY KEY ").append(buildIndexColumn(tableIndex)); + } else { + if (UNIQUE.equals(this)) { + script.append("CREATE UNIQUE INDEX "); + } else { + script.append("CREATE INDEX "); + } + script.append(buildIndexName(tableIndex)).append(" ON \"").append(tableIndex.getSchemaName()).append("\".\"").append(tableIndex.getTableName()).append("\" ").append(buildIndexColumn(tableIndex)); + } + return script.toString(); + } + + public String buildCreateIndexScript(TableIndex tableIndex) { + StringBuilder script = new StringBuilder(); + if (PRIMARY_KEY.equals(this)) { + script.append("CONSTRAINT ").append(tableIndex.getTableName()).append("_").append("PK PRIMARY KEY ").append(buildIndexColumn(tableIndex)); + } else { + if (UNIQUE.equals(this)) { + script.append("CREATE UNIQUE INDEX "); + } else { + script.append("CREATE INDEX "); + } + script.append(buildIndexName(tableIndex)).append(" ON \"").append(tableIndex.getSchemaName()).append("\".\"").append(tableIndex.getTableName()).append("\" ").append(buildIndexColumn(tableIndex)); + } + return script.toString(); + } + + private String buildIndexColumn(TableIndex tableIndex) { + StringBuilder script = new StringBuilder(); + script.append("("); + for (TableIndexColumn column : tableIndex.getColumnList()) { + if (StringUtils.isNotBlank(column.getColumnName())) { + script.append(column.getColumnName()); + if (!StringUtils.isBlank(column.getAscOrDesc()) && !PRIMARY_KEY.equals(this)) { + script.append(" ").append(column.getAscOrDesc()); + } + script.append(","); + } + } + script.deleteCharAt(script.length() - 1); + script.append(")"); + return script.toString(); + } + + private String buildIndexName(TableIndex tableIndex) { + return "\"" + tableIndex.getSchemaName() + "\"." + "\"" + tableIndex.getName() + "\""; + } + + public String buildModifyIndex(TableIndex tableIndex) { + if (EditStatus.DELETE.name().equals(tableIndex.getEditStatus())) { + return buildDropIndex(tableIndex); + } + if (EditStatus.MODIFY.name().equals(tableIndex.getEditStatus())) { + return StringUtils.join(buildDropIndex(tableIndex), ";\n", buildIndexScript(tableIndex)); + } + if (EditStatus.ADD.name().equals(tableIndex.getEditStatus())) { + return StringUtils.join(buildIndexScript(tableIndex)); + } + return ""; + } + + private String buildDropIndex(TableIndex tableIndex) { + if (DuckDBIndexTypeEnum.PRIMARY_KEY.getName().equals(tableIndex.getType())) { + String tableName = "" + tableIndex.getSchemaName() + "." + tableIndex.getTableName() ; + return StringUtils.join("ALTER TABLE ",tableName," DROP PRIMARY KEY"); + } + StringBuilder script = new StringBuilder(); + script.append("DROP INDEX "); + script.append(buildIndexName(tableIndex)); + + return script.toString(); + } + + public static List getIndexTypes() { + return Arrays.asList(DuckDBIndexTypeEnum.values()).stream().map(DuckDBIndexTypeEnum::getIndexType).collect(java.util.stream.Collectors.toList()); + } +} diff --git a/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/resources/META-INF/services/ai.chat2db.spi.Plugin b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/resources/META-INF/services/ai.chat2db.spi.Plugin new file mode 100644 index 000000000..46d0d0d83 --- /dev/null +++ b/chat2db-server/chat2db-plugins/chat2db-duckdb/src/main/resources/META-INF/services/ai.chat2db.spi.Plugin @@ -0,0 +1 @@ +ai.chat2db.plugin.duckdb.DuckDBPlugin \ No newline at end of file diff --git a/chat2db-server/chat2db-plugins/chat2db-h2/src/main/java/ai/chat2db/plugin/h2/H2Meta.java b/chat2db-server/chat2db-plugins/chat2db-h2/src/main/java/ai/chat2db/plugin/h2/H2Meta.java index 1b1cc2fd6..05f938c3b 100644 --- a/chat2db-server/chat2db-plugins/chat2db-h2/src/main/java/ai/chat2db/plugin/h2/H2Meta.java +++ b/chat2db-server/chat2db-plugins/chat2db-h2/src/main/java/ai/chat2db/plugin/h2/H2Meta.java @@ -3,6 +3,7 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; +import java.sql.SQLException; import java.util.*; import java.util.stream.Collectors; diff --git a/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/builder/HiveSqlBuilder.java b/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/builder/HiveSqlBuilder.java index 02a761df7..a3ca677dd 100644 --- a/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/builder/HiveSqlBuilder.java +++ b/chat2db-server/chat2db-plugins/chat2db-hive/src/main/java/ai/chat2db/plugin/hive/builder/HiveSqlBuilder.java @@ -2,6 +2,7 @@ import ai.chat2db.plugin.hive.type.HiveColumnTypeEnum; import ai.chat2db.plugin.hive.type.HiveIndexTypeEnum; +import ai.chat2db.spi.SqlBuilder; import ai.chat2db.spi.jdbc.DefaultSqlBuilder; import ai.chat2db.spi.model.Database; import ai.chat2db.spi.model.Table; @@ -9,6 +10,7 @@ import ai.chat2db.spi.model.TableIndex; import org.apache.commons.lang3.StringUtils; +import java.util.List; public class HiveSqlBuilder extends DefaultSqlBuilder implements SqlBuilder
{ diff --git a/chat2db-server/chat2db-plugins/pom.xml b/chat2db-server/chat2db-plugins/pom.xml index a98b555cb..878eddd17 100644 --- a/chat2db-server/chat2db-plugins/pom.xml +++ b/chat2db-server/chat2db-plugins/pom.xml @@ -30,6 +30,7 @@ chat2db-hivechat2db-kingbasechat2db-timeplus + chat2db-duckdb diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/pom.xml b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/pom.xml index 39d5c2a2d..7fdb052b2 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/pom.xml +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/pom.xml @@ -126,6 +126,11 @@ chat2db-timeplus${revision} + + ai.chat2db + chat2db-duckdb + ${version} + commons-codec commons-codec diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java index 18038e95a..9834cdf10 100644 --- a/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-core/src/main/java/ai/chat2db/server/domain/core/impl/TableServiceImpl.java @@ -424,7 +424,7 @@ private long addDBCache(Long dataSourceId, String databaseName, String schemaNam Connection connection = Chat2DBContext.getConnection(); long n = 0; try (ResultSet resultSet = connection.getMetaData().getTables(databaseName, schemaName, null, - new String[]{"TABLE", "SYSTEM TABLE"})) { + new String[]{"TABLE", "SYSTEM TABLE", "BASE TABLE"})) { List cacheDOS = new ArrayList<>(); while (resultSet.next()) { TableCacheDO tableCacheDO = new TableCacheDO(); diff --git a/chat2db-server/chat2db-server-domain/chat2db-server-domain-repository/src/main/resources/db/migration/V2_1_10__REMOVEdEMO.sql b/chat2db-server/chat2db-server-domain/chat2db-server-domain-repository/src/main/resources/db/migration/V2_1_10__REMOVEdEMO.sql new file mode 100644 index 000000000..b0f5f6300 --- /dev/null +++ b/chat2db-server/chat2db-server-domain/chat2db-server-domain-repository/src/main/resources/db/migration/V2_1_10__REMOVEdEMO.sql @@ -0,0 +1,7 @@ +delete from DATA_SOURCE where ALIAS ='DEMO@db.sqlgpt.cn'; + +delete from DASHBOARD where id =ID; + +delete from CHART where id<=3; + +delete from DASHBOARD_CHART_RELATION where CHART_ID<=3; diff --git a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java index f9c529a5b..7881cf93a 100644 --- a/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java +++ b/chat2db-server/chat2db-server-web/chat2db-server-web-api/src/main/java/ai/chat2db/server/web/api/controller/rdb/converter/RdbWebConverter.java @@ -42,6 +42,8 @@ public abstract class RdbWebConverter { public abstract DlExecuteParam request2param(DmlRequest request); + + public abstract GroupByParam request2param(GroupByRequest request); /** * Parameter conversion *