Skip to content

Commit 487adbc

Browse files
alex268alex268
committed
Added flag to forced usage of BulkUpsert and ScanQuery
1 parent b3a73e0 commit 487adbc

File tree

7 files changed

+141
-18
lines changed

7 files changed

+141
-18
lines changed

‎jdbc/src/main/java/tech/ydb/jdbc/query/YdbQuery.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,21 @@ public static YdbQuery parseQuery(String query, YdbQueryProperties opts) throws
6969
QueryType type = opts.getForcedQueryType();
7070
if (type == null) {
7171
type = parser.detectQueryType();
72+
73+
if (opts.isForcedScanAndBulks() && type != QueryType.SCHEME_QUERY && type != QueryType.EXPLAIN_QUERY) {
74+
if (parser.getYqlBatcher().isValidBatch()) {
75+
parser.getYqlBatcher().setForcedUpsert();
76+
type = QueryType.BULK_QUERY;
77+
} else {
78+
type = QueryType.SCAN_QUERY;
79+
}
80+
}
7281
}
82+
7383
if (parser.getYqlBatcher().isValidBatch()) {
7484
return new YdbQuery(query, preparedYQL, parser.getStatements(), parser.getYqlBatcher(), type);
7585
}
86+
7687
return new YdbQuery(query, preparedYQL, parser.getStatements(), type);
7788
}
7889
}

‎jdbc/src/main/java/tech/ydb/jdbc/query/YqlBatcher.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ private enum State {
3434
private final List<String> columns = new ArrayList<>();
3535
private final List<String> values = new ArrayList<>();
3636

37+
public void setForcedUpsert() {
38+
cmd = Cmd.UPSERT;
39+
}
40+
3741
public boolean isInsert() {
3842
return cmd == Cmd.INSERT;
3943
}

‎jdbc/src/main/java/tech/ydb/jdbc/settings/YdbConfig.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,14 +178,14 @@ public DriverPropertyInfo[] toPropertyInfo() throws SQLException {
178178
YdbOperationProperties.SCHEME_QUERY_TX_MODE.toInfo(properties),
179179
YdbOperationProperties.SCAN_QUERY_TX_MODE.toInfo(properties),
180180
YdbOperationProperties.BULK_QUERY_TX_MODE.toInfo(properties),
181-
YdbOperationProperties.FORCE_SCAN_BULKS.toInfo(properties),
182181

183182
YdbQueryProperties.DISABLE_PREPARE_DATAQUERY.toInfo(properties),
184183
YdbQueryProperties.DISABLE_AUTO_PREPARED_BATCHES.toInfo(properties),
185184
YdbQueryProperties.DISABLE_DETECT_SQL_OPERATIONS.toInfo(properties),
186185
YdbQueryProperties.DISABLE_JDBC_PARAMETERS.toInfo(properties),
187186
YdbQueryProperties.DISABLE_JDBC_PARAMETERS_DECLARE.toInfo(properties),
188187
YdbQueryProperties.FORCE_QUERY_MODE.toInfo(properties),
188+
YdbQueryProperties.FORCE_SCAN_BULKS.toInfo(properties),
189189
};
190190
}
191191

‎jdbc/src/main/java/tech/ydb/jdbc/settings/YdbOperationProperties.java‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,6 @@ public class YdbOperationProperties {
5353
FakeTxMode.ERROR
5454
);
5555

56-
static final YdbProperty<Boolean> FORCE_SCAN_BULKS = YdbProperty.bool("forceScanAndBulk",
57-
"Force usage of bulk upserts instead of upserts/inserts and scan query for selects",
58-
false
59-
);
60-
6156
private static final int MAX_ROWS = 1000; // TODO: how to figure out the max rows of current connection?
6257

6358
private final YdbValue<Duration> joinDuration;
@@ -73,8 +68,6 @@ public class YdbOperationProperties {
7368
private final YdbValue<FakeTxMode> schemeQueryTxMode;
7469
private final YdbValue<FakeTxMode> bulkQueryTxMode;
7570

76-
private final YdbValue<Boolean> forceScanBulks;
77-
7871
public YdbOperationProperties(YdbConfig config) throws SQLException {
7972
Properties props = config.getProperties();
8073

@@ -90,8 +83,6 @@ public YdbOperationProperties(YdbConfig config) throws SQLException {
9083
this.scanQueryTxMode = SCAN_QUERY_TX_MODE.readValue(props);
9184
this.schemeQueryTxMode = SCHEME_QUERY_TX_MODE.readValue(props);
9285
this.bulkQueryTxMode = BULK_QUERY_TX_MODE.readValue(props);
93-
94-
this.forceScanBulks = FORCE_SCAN_BULKS.readValue(props);
9586
}
9687

9788
public Duration getJoinDuration() {
@@ -122,10 +113,6 @@ public FakeTxMode getBulkQueryTxMode() {
122113
return bulkQueryTxMode.getValue();
123114
}
124115

125-
public boolean getForceScanBulks() {
126-
return forceScanBulks.getValue();
127-
}
128-
129116
public Duration getSessionTimeout() {
130117
return sessionTimeout.getValue();
131118
}

‎jdbc/src/main/java/tech/ydb/jdbc/settings/YdbQueryProperties.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ public class YdbQueryProperties {
3030
static final YdbProperty<QueryType> FORCE_QUERY_MODE = YdbProperty.enums("forceQueryMode", QueryType.class,
3131
"Force usage one of query modes (DATA_QUERY, SCAN_QUERY, SCHEME_QUERY or EXPLAIN_QUERYn) for all statements"
3232
);
33+
static final YdbProperty<Boolean> FORCE_SCAN_BULKS = YdbProperty.bool("forceScanAndBulk",
34+
"Force usage of bulk upserts instead of upserts/inserts and scan query for selects",
35+
false
36+
);
3337

3438
private final boolean isDetectQueryType;
3539
private final boolean isDetectJdbcParameters;
@@ -39,6 +43,7 @@ public class YdbQueryProperties {
3943
private final boolean isDetectBatchQueries;
4044

4145
private final QueryType forcedType;
46+
private final boolean isForcedScanAndBulks;
4247

4348
public YdbQueryProperties(YdbConfig config) throws SQLException {
4449
Properties props = config.getProperties();
@@ -59,6 +64,7 @@ public YdbQueryProperties(YdbConfig config) throws SQLException {
5964
&& !disableJdbcParametersDeclare;
6065

6166
this.forcedType = FORCE_QUERY_MODE.readValue(props).getValue();
67+
this.isForcedScanAndBulks = FORCE_SCAN_BULKS.readValue(props).getValue();
6268
}
6369

6470
public boolean isDetectQueryType() {
@@ -84,4 +90,8 @@ public boolean isDetectBatchQueries() {
8490
public QueryType getForcedQueryType() {
8591
return forcedType;
8692
}
93+
94+
public boolean isForcedScanAndBulks() {
95+
return isForcedScanAndBulks;
96+
}
8797
}

‎jdbc/src/test/java/tech/ydb/jdbc/YdbDriverColumnTablesTest.java‎

Lines changed: 113 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ public class YdbDriverColumnTablesTest {
3131
"Data manipulation queries do not support column shard tables. (S_ERROR)";
3232
private final static String ERROR_TRANSACTIONS =
3333
"Transactions between column and row tables are disabled at current time. (S_ERROR)";
34+
private final static String ERROR_SCAN_QUERY =
35+
"Scan query should have a single result set. (S_ERROR)";
36+
3437
private final static String ERROR_BULK_UNSUPPORTED =
3538
"BULK mode is available only for prepared statement with one UPSERT";
3639

@@ -88,7 +91,7 @@ public void defaultModeTest() throws SQLException {
8891
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
8992
ps.addBatch();
9093
}
91-
ExceptionAssert.ydbException(ERROR_TRANSACTIONS, ps::executeBatch);
94+
ExceptionAssert.ydbException(ERROR_DATA_MANIPULATION, ps::executeBatch);
9295
}
9396

9497
// batch insert
@@ -99,7 +102,7 @@ public void defaultModeTest() throws SQLException {
99102
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
100103
ps.addBatch();
101104
}
102-
ExceptionAssert.ydbException(ERROR_TRANSACTIONS, ps::executeBatch);
105+
ExceptionAssert.ydbException(ERROR_DATA_MANIPULATION, ps::executeBatch);
103106
}
104107

105108
// read all
@@ -208,4 +211,112 @@ public void prefixesTest() throws SQLException {
208211
ExceptionAssert.sqlException(ERROR_BULK_UNSUPPORTED, () -> conn.prepareStatement("BULK " + DELETE_ROW));
209212
}
210213
}
214+
215+
@Test
216+
public void forceScanAndBulkTest() throws SQLException {
217+
try (Connection conn = DriverManager.getConnection(jdbcURL.withArg("forceScanAndBulk", "true").build())) {
218+
try {
219+
conn.createStatement().execute(DROP_TABLE);
220+
} catch (SQLException e) {
221+
// ignore
222+
}
223+
224+
conn.createStatement().execute(CREATE_TABLE);
225+
226+
LocalDate ld = LocalDate.of(2017, 12, 3);
227+
String prefix = "text-value-";
228+
int idx = 0;
229+
230+
// single bulk upsert
231+
try (PreparedStatement ps = conn.prepareStatement(UPSERT_ROW)) {
232+
ps.setInt(1, ++idx);
233+
ps.setString(2, prefix + idx);
234+
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
235+
ps.executeUpdate();
236+
}
237+
238+
// single bulk insert
239+
try (PreparedStatement ps = conn.prepareStatement(INSERT_ROW)) {
240+
ps.setInt(1, ++idx);
241+
ps.setString(2, prefix + idx);
242+
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
243+
ps.executeUpdate();
244+
}
245+
246+
// scan read
247+
try (Statement st = conn.createStatement()) {
248+
int readed = 0;
249+
try (ResultSet rs = st.executeQuery(SELECT_ALL)) {
250+
while (rs.next()) {
251+
readed++;
252+
Assertions.assertEquals(readed, rs.getInt("id"));
253+
Assertions.assertEquals(prefix + readed, rs.getString("value"));
254+
Assertions.assertEquals(Date.valueOf(ld.plusDays(readed)), rs.getDate("date"));
255+
}
256+
}
257+
Assertions.assertEquals(2, readed);
258+
}
259+
260+
// batch bulk upsert
261+
try (PreparedStatement ps = conn.prepareStatement(UPSERT_ROW)) {
262+
for (int j = 0; j < 2000; j++) {
263+
ps.setInt(1, ++idx);
264+
ps.setString(2, prefix + idx);
265+
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
266+
ps.addBatch();
267+
}
268+
ps.executeBatch();
269+
270+
// single row upsert
271+
ps.setInt(1, ++idx);
272+
ps.setString(2, prefix + idx);
273+
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
274+
ps.execute();
275+
}
276+
277+
// batch bulk inserts
278+
try (PreparedStatement ps = conn.prepareStatement(INSERT_ROW)) {
279+
for (int j = 0; j < 2000; j++) {
280+
ps.setInt(1, ++idx);
281+
ps.setString(2, prefix + idx);
282+
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
283+
ps.addBatch();
284+
}
285+
ps.executeBatch();
286+
287+
// single row insert
288+
ps.setInt(1, ++idx);
289+
ps.setString(2, prefix + idx);
290+
ps.setDate(3, Date.valueOf(ld.plusDays(idx)));
291+
ps.execute();
292+
}
293+
294+
// read all
295+
try (Statement st = conn.createStatement()) {
296+
int readed = 0;
297+
try (ResultSet rs = st.executeQuery(SELECT_ALL)) {
298+
while (rs.next()) {
299+
readed++;
300+
Assertions.assertEquals(readed, rs.getInt("id"));
301+
Assertions.assertEquals(prefix + readed, rs.getString("value"));
302+
Assertions.assertEquals(Date.valueOf(ld.plusDays(readed)), rs.getDate("date"));
303+
}
304+
}
305+
Assertions.assertEquals(4004, readed);
306+
}
307+
308+
// single update
309+
try (PreparedStatement ps = conn.prepareStatement(UPDATE_ROW)) {
310+
ps.setString(1, "updated-value");
311+
ps.setInt(2, 1);
312+
ExceptionAssert.ydbException(ERROR_SCAN_QUERY, ps::execute);
313+
}
314+
315+
// single delete
316+
try (PreparedStatement ps = conn.prepareStatement(DELETE_ROW)) {
317+
ps.setInt(1, 2);
318+
ExceptionAssert.ydbException(ERROR_SCAN_QUERY, ps::execute);
319+
}
320+
}
321+
}
211322
}

‎jdbc/src/test/java/tech/ydb/jdbc/settings/YdbDriverProperitesTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -329,13 +329,13 @@ static DriverPropertyInfo[] defaultPropertyInfo(@Nullable String localDatacenter
329329
new DriverPropertyInfo("schemeQueryTxMode", "ERROR"),
330330
new DriverPropertyInfo("scanQueryTxMode", "ERROR"),
331331
new DriverPropertyInfo("bulkUpsertQueryTxMode", "ERROR"),
332-
new DriverPropertyInfo("forceScanAndBulk", "false"),
333332
new DriverPropertyInfo("disablePrepareDataQuery", "false"),
334333
new DriverPropertyInfo("disableAutoPreparedBatches", "false"),
335334
new DriverPropertyInfo("disableDetectSqlOperations", "false"),
336335
new DriverPropertyInfo("disableJdbcParameters", "false"),
337336
new DriverPropertyInfo("disableJdbcParameterDeclare", "false"),
338337
new DriverPropertyInfo("forceQueryMode", ""),
338+
new DriverPropertyInfo("forceScanAndBulk", "false"),
339339
};
340340
}
341341

@@ -368,13 +368,13 @@ static DriverPropertyInfo[] customizedPropertyInfo() {
368368
new DriverPropertyInfo("schemeQueryTxMode", "SHADOW_COMMIT"),
369369
new DriverPropertyInfo("scanQueryTxMode", "FAKE_TX"),
370370
new DriverPropertyInfo("bulkUpsertQueryTxMode", "SHADOW_COMMIT"),
371-
new DriverPropertyInfo("forceScanAndBulk", "true"),
372371
new DriverPropertyInfo("disablePrepareDataQuery", "true"),
373372
new DriverPropertyInfo("disableAutoPreparedBatches", "true"),
374373
new DriverPropertyInfo("disableDetectSqlOperations", "true"),
375374
new DriverPropertyInfo("disableJdbcParameters", "true"),
376375
new DriverPropertyInfo("disableJdbcParameterDeclare", "true"),
377376
new DriverPropertyInfo("forceQueryMode", "SCAN_QUERY"),
377+
new DriverPropertyInfo("forceScanAndBulk", "true"),
378378
};
379379
}
380380

0 commit comments

Comments
 (0)