diff --git a/jdbc/src/main/java/tech/ydb/jdbc/query/YdbQuery.java b/jdbc/src/main/java/tech/ydb/jdbc/query/YdbQuery.java index 31e7f4ec..ccc0fb83 100644 --- a/jdbc/src/main/java/tech/ydb/jdbc/query/YdbQuery.java +++ b/jdbc/src/main/java/tech/ydb/jdbc/query/YdbQuery.java @@ -15,21 +15,17 @@ public class YdbQuery { private final String originQuery; private final String preparedYQL; private final List statements; - private final YqlBatcher batch; + private final YqlBatcher batcher; private final QueryType type; private final boolean isPlainYQL; - YdbQuery(String originQuery, String preparedYQL, List stats, QueryType type) { - this(originQuery, preparedYQL, stats, null, type); - } - - YdbQuery(String originQuery, String preparedYQL, List stats, YqlBatcher batch, QueryType type) { + YdbQuery(String originQuery, String preparedYQL, List stats, YqlBatcher batcher, QueryType type) { this.originQuery = originQuery; this.preparedYQL = preparedYQL; this.statements = stats; this.type = type; - this.batch = batch; + this.batcher = batcher; boolean hasJdbcParamters = false; for (QueryStatement st: statements) { @@ -43,7 +39,7 @@ public QueryType getType() { } public YqlBatcher getYqlBatcher() { - return batch; + return batcher.isValidBatch() ? batcher : null; } public boolean isPlainYQL() { @@ -66,32 +62,29 @@ public static YdbQuery parseQuery(String query, YdbQueryProperties opts) throws YdbQueryParser parser = new YdbQueryParser(opts.isDetectQueryType(), opts.isDetectJdbcParameters()); String preparedYQL = parser.parseSQL(query); - QueryType type = opts.getForcedQueryType(); - if (type == null) { - type = parser.detectQueryType(); - YqlBatcher batcher = parser.getYqlBatcher(); - - if (opts.isForcedScanAndBulks()) { - if (batcher.isValidBatch()) { - if (batcher.getCommand() == YqlBatcher.Cmd.UPSERT) { - type = QueryType.BULK_QUERY; - } - if (batcher.getCommand() == YqlBatcher.Cmd.INSERT) { - parser.getYqlBatcher().setForcedUpsert(); - type = QueryType.BULK_QUERY; - } - } + QueryType type = null; + YqlBatcher batcher = parser.getYqlBatcher(); + List statements = parser.getStatements(); - if (parser.getStatements().size() == 1 && parser.getStatements().get(0).getCmd() == QueryCmd.SELECT) { + if (batcher.isValidBatch()) { + if (batcher.getCommand() == YqlBatcher.Cmd.INSERT && opts.isReplaceInsertToUpsert()) { + batcher.setForcedUpsert(); + } + if (batcher.getCommand() == YqlBatcher.Cmd.UPSERT && opts.isForceBulkUpsert()) { + type = QueryType.BULK_QUERY; + } + } else { + if (opts.isForceScanSelect() && statements.size() == 1 && statements.get(0).getCmd() == QueryCmd.SELECT) { + if (parser.detectQueryType() == QueryType.DATA_QUERY) { // Only data queries may be converter to SCAN type = QueryType.SCAN_QUERY; } } } - if (parser.getYqlBatcher().isValidBatch()) { - return new YdbQuery(query, preparedYQL, parser.getStatements(), parser.getYqlBatcher(), type); + if (type == null) { + type = parser.detectQueryType(); } - return new YdbQuery(query, preparedYQL, parser.getStatements(), type); + return new YdbQuery(query, preparedYQL, statements, batcher, type); } } diff --git a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConfig.java b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConfig.java index 1e0d3c8b..4411fd22 100644 --- a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConfig.java +++ b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConfig.java @@ -184,8 +184,10 @@ public DriverPropertyInfo[] toPropertyInfo() throws SQLException { YdbQueryProperties.DISABLE_DETECT_SQL_OPERATIONS.toInfo(properties), YdbQueryProperties.DISABLE_JDBC_PARAMETERS.toInfo(properties), YdbQueryProperties.DISABLE_JDBC_PARAMETERS_DECLARE.toInfo(properties), - YdbQueryProperties.FORCE_QUERY_MODE.toInfo(properties), - YdbQueryProperties.FORCE_SCAN_BULKS.toInfo(properties), + + YdbQueryProperties.REPLACE_INSERT_TO_UPSERT.toInfo(properties), + YdbQueryProperties.FORCE_BULK_UPSERT.toInfo(properties), + YdbQueryProperties.FORCE_SCAN_SELECT.toInfo(properties), }; } diff --git a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbQueryProperties.java b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbQueryProperties.java index 80d09619..78d51073 100644 --- a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbQueryProperties.java +++ b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbQueryProperties.java @@ -2,7 +2,9 @@ import java.sql.SQLException; import java.util.Properties; +import java.util.logging.Logger; +import tech.ydb.jdbc.YdbDriver; import tech.ydb.jdbc.query.QueryType; @@ -12,6 +14,8 @@ * @author Aleksandr Gorshenin */ public class YdbQueryProperties { + private static final Logger LOGGER = Logger.getLogger(YdbDriver.class.getName()); + static final YdbProperty DISABLE_DETECT_SQL_OPERATIONS = YdbProperty.bool("disableDetectSqlOperations", "Disable detecting SQL operation based on SQL keywords", false); @@ -27,14 +31,23 @@ public class YdbQueryProperties { static final YdbProperty DISABLE_JDBC_PARAMETERS_DECLARE = YdbProperty.bool("disableJdbcParameterDeclare", "Disable enforce DECLARE section for JDBC parameters '?'", false); - static final YdbProperty FORCE_QUERY_MODE = YdbProperty.enums("forceQueryMode", QueryType.class, - "Force usage one of query modes (DATA_QUERY, SCAN_QUERY, SCHEME_QUERY or EXPLAIN_QUERYn) for all statements" + @Deprecated + private static final YdbProperty FORCE_QUERY_MODE = YdbProperty.enums("forceQueryMode", QueryType.class, + "Force usage one of query modes (DATA_QUERY, SCAN_QUERY, SCHEME_QUERY or EXPLAIN_QUERY) for all statements" ); - static final YdbProperty FORCE_SCAN_BULKS = YdbProperty.bool("forceScanAndBulk", + @Deprecated + private static final YdbProperty FORCE_SCAN_BULKS = YdbProperty.bool("forceScanAndBulk", "Force usage of bulk upserts instead of upserts/inserts and scan query for selects", false ); + static final YdbProperty REPLACE_INSERT_TO_UPSERT = YdbProperty.bool("replaceInsertByUpsert", + "Convert all INSERT statements to UPSERT statements", false); + static final YdbProperty FORCE_BULK_UPSERT = YdbProperty.bool("forceBulkUpsert", + "Execute all UPSERT statements as BulkUpserts", false); + static final YdbProperty FORCE_SCAN_SELECT = YdbProperty.bool("forceScanSelect", + "Execute all SELECT statements as ScanQuery", false); + private final boolean isDetectQueryType; private final boolean isDetectJdbcParameters; private final boolean isDeclareJdbcParameters; @@ -42,8 +55,9 @@ public class YdbQueryProperties { private final boolean isPrepareDataQueries; private final boolean isDetectBatchQueries; - private final QueryType forcedType; - private final boolean isForcedScanAndBulks; + private final boolean isReplaceInsertToUpsert; + private final boolean isForceBulkUpsert; + private final boolean isForceScanSelect; public YdbQueryProperties(YdbConfig config) throws SQLException { Properties props = config.getProperties(); @@ -63,8 +77,24 @@ public YdbQueryProperties(YdbConfig config) throws SQLException { this.isDeclareJdbcParameters = !disableSqlOperationsDetect && !disableJdbcParametersParse && !disableJdbcParametersDeclare; - this.forcedType = FORCE_QUERY_MODE.readValue(props).getValue(); - this.isForcedScanAndBulks = FORCE_SCAN_BULKS.readValue(props).getValue(); + + YdbValue forcedType = FORCE_QUERY_MODE.readValue(props); + if (forcedType.hasValue()) { + LOGGER.warning("Option 'forceQueryMode' is deprecated and will be removed in next versions. " + + "Use options 'forceScanSelect' and 'forceBulkUpsert' instead"); + } + YdbValue forceScanAndBulk = FORCE_SCAN_BULKS.readValue(props); + if (forceScanAndBulk.hasValue()) { + LOGGER.warning("Option 'forceScanAndBulk' is deprecated and will be removed in next versions. " + + "Use options 'replaceInsertByUpsert', 'forceScanSelect' and 'forceBulkUpsert' instead"); + } + + this.isReplaceInsertToUpsert = REPLACE_INSERT_TO_UPSERT.readValue(props) + .getValueOrOther(forceScanAndBulk.getValue()); + this.isForceBulkUpsert = FORCE_BULK_UPSERT.readValue(props) + .getValueOrOther(forceScanAndBulk.getValue() || forcedType.getValue() == QueryType.BULK_QUERY); + this.isForceScanSelect = FORCE_SCAN_SELECT.readValue(props) + .getValueOrOther(forceScanAndBulk.getValue() || forcedType.getValue() == QueryType.SCAN_QUERY); } public boolean isDetectQueryType() { @@ -87,11 +117,15 @@ public boolean isDetectBatchQueries() { return isDetectBatchQueries; } - public QueryType getForcedQueryType() { - return forcedType; + public boolean isReplaceInsertToUpsert() { + return isReplaceInsertToUpsert; + } + + public boolean isForceBulkUpsert() { + return isForceBulkUpsert; } - public boolean isForcedScanAndBulks() { - return isForcedScanAndBulks; + public boolean isForceScanSelect() { + return isForceScanSelect; } } diff --git a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbValue.java b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbValue.java index d0508539..05200dd1 100644 --- a/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbValue.java +++ b/jdbc/src/main/java/tech/ydb/jdbc/settings/YdbValue.java @@ -17,6 +17,10 @@ public T getValue() { return value; } + public T getValueOrOther(T other) { + return isPresent ? value : other; + } + public boolean hasValue() { return isPresent; } diff --git a/jdbc/src/test/java/tech/ydb/jdbc/YdbDriverOlapTablesTest.java b/jdbc/src/test/java/tech/ydb/jdbc/YdbDriverOlapTablesTest.java index 442fe9f5..d501a102 100644 --- a/jdbc/src/test/java/tech/ydb/jdbc/YdbDriverOlapTablesTest.java +++ b/jdbc/src/test/java/tech/ydb/jdbc/YdbDriverOlapTablesTest.java @@ -216,7 +216,12 @@ public void customQueriesTest() throws SQLException { @Test public void forceScanAndBulkTest() throws SQLException { - try (Connection conn = DriverManager.getConnection(jdbcURL.withArg("forceScanAndBulk", "true").build())) { + try (Connection conn = DriverManager.getConnection(jdbcURL + .withArg("replaceInsertByUpsert", "true") + .withArg("forceBulkUpsert", "true") + .withArg("forceScanSelect", "true") + .build() + )) { try { conn.createStatement().execute(DROP_TABLE); } catch (SQLException e) { diff --git a/jdbc/src/test/java/tech/ydb/jdbc/YdbDriverTablesTest.java b/jdbc/src/test/java/tech/ydb/jdbc/YdbDriverTablesTest.java index 143de353..79828585 100644 --- a/jdbc/src/test/java/tech/ydb/jdbc/YdbDriverTablesTest.java +++ b/jdbc/src/test/java/tech/ydb/jdbc/YdbDriverTablesTest.java @@ -28,9 +28,6 @@ public class YdbDriverTablesTest { private static final JdbcUrlHelper jdbcURL = new JdbcUrlHelper(ydb); - private final static String ERROR_SCAN_QUERY = - "Scan query should have a single result set. (S_ERROR)"; - private final static String ERROR_BULK_UNSUPPORTED = "BULK mode is available only for prepared statement with one UPSERT"; @@ -206,7 +203,12 @@ public void customQueriesTest() throws SQLException { @Test public void forceScanAndBulkTest() throws SQLException { - try (Connection conn = DriverManager.getConnection(jdbcURL.withArg("forceScanAndBulk", "true").build())) { + try (Connection conn = DriverManager.getConnection(jdbcURL + .withArg("replaceInsertByUpsert", "true") + .withArg("forceBulkUpsert", "true") + .withArg("forceScanSelect", "true") + .build() + )) { try { conn.createStatement().execute(DROP_TABLE); } catch (SQLException e) { diff --git a/jdbc/src/test/java/tech/ydb/jdbc/query/YdbQueryTest.java b/jdbc/src/test/java/tech/ydb/jdbc/query/YdbQueryTest.java index f4940cef..df1a34a9 100644 --- a/jdbc/src/test/java/tech/ydb/jdbc/query/YdbQueryTest.java +++ b/jdbc/src/test/java/tech/ydb/jdbc/query/YdbQueryTest.java @@ -112,68 +112,136 @@ public void mixQueryExceptionTest() throws SQLException { } @Test - public void forsedTypeTest() throws SQLException { + public void forsedScanSelects() throws SQLException { YdbQueryProperties opts = new ParamsBuilder() - .with("forceQueryMode", "SCHEME_QUERY") + .with("forceScanSelect", "true") .build(); Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, "CREATE TABLE test_table (id int, value text)" )); - Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, - "\tcreate TABLE test_table2 (id int, value text);" - )); - Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, " drop TABLE test_table1 (id int, value text);" + "ALTER TABLE test_table2 (id int, value text);" )); - Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, + Assertions.assertEquals(QueryType.SCAN_QUERY, parsedQueryType(opts, "SELECT id, value FROM test_table" )); - Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, + Assertions.assertEquals(QueryType.SCAN_QUERY, parsedQueryType(opts, + "SCAN SELECT id, value FROM test_table" + )); + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, + "SELECT id, value FROM test_table;" + + "SELECT id, value FROM test_table2;" + )); + + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, "UPSERT INTO test_table VALUES (?, ?)" )); - Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, "DELETE FROM test_table" )); - Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, "SELECT id, value FROM test_table;\n" + "UPSERT INTO test_table VALUES (?, ?);" + "DELETE FROM test_table" )); - Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, - "SELECT id, value FROM test_table;\n" + - "UPDATE test_table SET value = ? WHERE id = ?;" + - "SELECT id, value FROM test_table WHERE id=CREATE" + Assertions.assertEquals(QueryType.EXPLAIN_QUERY, parsedQueryType(opts, + "EXPLAIN SELECT id, value FROM test_table" )); + } + + @Test + public void forsedBulkUpsert() throws SQLException { + YdbQueryProperties opts = new ParamsBuilder() + .with("forceBulkUpsert", "true") + .build(); Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, + "CREATE TABLE test_table (id int, value text)" + )); + Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, + " drop TABLE test_table1 (id int, value text);" + + "ALTER TABLE test_table2 (id int, value text);" + )); + + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, + "SELECT id, value FROM test_table" + )); + Assertions.assertEquals(QueryType.SCAN_QUERY, parsedQueryType(opts, "SCAN SELECT id, value FROM test_table" )); + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, + "SELECT id, value FROM test_table;" + + "SELECT id, value FROM test_table2;" + )); + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, + "INSERT INTO test_table(id, value) VALUES (?, ?)" + )); + Assertions.assertEquals(QueryType.BULK_QUERY, parsedQueryType(opts, + "UPSERT INTO test_table(id, value) VALUES (?, ?)" + )); - Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, + "DELETE FROM test_table" + )); + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, + "SELECT id, value FROM test_table;\n" + + "UPSERT INTO test_table VALUES (?, ?);" + + "DELETE FROM test_table" + )); + + Assertions.assertEquals(QueryType.EXPLAIN_QUERY, parsedQueryType(opts, "EXPLAIN SELECT id, value FROM test_table" )); + } + + @Test + public void forcedUpsert() throws SQLException { + YdbQueryProperties opts = new ParamsBuilder() + .with("replaceInsertByUpsert", "true") + .with("forceBulkUpsert", "true") + .build(); Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, - "CREATE TABLE test_table (id int, value text);" + - "SELECT * FROM test_table;" + "CREATE TABLE test_table (id int, value text)" )); - Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, - "DROP TABLE test_table (id int, value text);SELECT * FROM test_table;" + " drop TABLE test_table1 (id int, value text);" + + "ALTER TABLE test_table2 (id int, value text);" )); - Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, - "SELECT * FROM test_table;CREATE TABLE test_table (id int, value text);" + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, + "SELECT id, value FROM test_table" + )); + Assertions.assertEquals(QueryType.SCAN_QUERY, parsedQueryType(opts, + "SCAN SELECT id, value FROM test_table" + )); + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, + "SELECT id, value FROM test_table;" + + "SELECT id, value FROM test_table2;" )); - Assertions.assertEquals(QueryType.SCHEME_QUERY, parsedQueryType(opts, - "SELECT * FROM test_table;\n\tCREATE TABLE test_table (id int, value text);" + Assertions.assertEquals(QueryType.BULK_QUERY, parsedQueryType(opts, + "INSERT INTO test_table(id, value) VALUES (?, ?)" + )); + Assertions.assertEquals(QueryType.BULK_QUERY, parsedQueryType(opts, + "UPSERT INTO test_table(id, value) VALUES (?, ?)" )); - } + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, + "DELETE FROM test_table" + )); + Assertions.assertEquals(QueryType.DATA_QUERY, parsedQueryType(opts, + "SELECT id, value FROM test_table;\n" + + "UPSERT INTO test_table VALUES (?, ?);" + + "DELETE FROM test_table" + )); + + Assertions.assertEquals(QueryType.EXPLAIN_QUERY, parsedQueryType(opts, + "EXPLAIN SELECT id, value FROM test_table" + )); + } } diff --git a/jdbc/src/test/java/tech/ydb/jdbc/settings/YdbDriverProperitesTest.java b/jdbc/src/test/java/tech/ydb/jdbc/settings/YdbDriverProperitesTest.java index 32bf1bc3..ae52b9ae 100644 --- a/jdbc/src/test/java/tech/ydb/jdbc/settings/YdbDriverProperitesTest.java +++ b/jdbc/src/test/java/tech/ydb/jdbc/settings/YdbDriverProperitesTest.java @@ -334,8 +334,9 @@ static DriverPropertyInfo[] defaultPropertyInfo(@Nullable String localDatacenter new DriverPropertyInfo("disableDetectSqlOperations", "false"), new DriverPropertyInfo("disableJdbcParameters", "false"), new DriverPropertyInfo("disableJdbcParameterDeclare", "false"), - new DriverPropertyInfo("forceQueryMode", ""), - new DriverPropertyInfo("forceScanAndBulk", "false"), + new DriverPropertyInfo("replaceInsertByUpsert", "false"), + new DriverPropertyInfo("forceBulkUpsert", "false"), + new DriverPropertyInfo("forceScanSelect", "false"), }; } @@ -373,8 +374,9 @@ static DriverPropertyInfo[] customizedPropertyInfo() { new DriverPropertyInfo("disableDetectSqlOperations", "true"), new DriverPropertyInfo("disableJdbcParameters", "true"), new DriverPropertyInfo("disableJdbcParameterDeclare", "true"), - new DriverPropertyInfo("forceQueryMode", "SCAN_QUERY"), - new DriverPropertyInfo("forceScanAndBulk", "true"), + new DriverPropertyInfo("replaceInsertByUpsert", "true"), + new DriverPropertyInfo("forceBulkUpsert", "true"), + new DriverPropertyInfo("forceScanSelect", "true"), }; }