Skip to content

Commit 8f2fde0

Browse files
muskan124947David-Engel
muskan124947
and
David-Engel
authored
Addressed performance issue on Bulk Copy API with batch insert (#2735)
* Addressed performance issue on Bulk Copy API with batch insert * Addressed comment * Update BatchExecutionWithBulkCopyTest.java * Few more test combinations --------- Co-authored-by: David Engel <[email protected]>
1 parent c734710 commit 8f2fde0

File tree

3 files changed

+233
-56
lines changed

3 files changed

+233
-56
lines changed

‎src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkBatchInsertRecord.java

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -186,18 +186,7 @@ else if (dateTimeFormatter != null)
186186
case Types.LONGVARCHAR:
187187
case Types.NCHAR:
188188
case Types.NVARCHAR:
189-
case Types.LONGNVARCHAR: {
190-
/*
191-
* If string data comes in as a byte array through setString (and sendStringParametersAsUnicode = false)
192-
* through Bulk Copy for Batch Insert API, convert the byte array to a string.
193-
* If the data is already a string, return it as is.
194-
*/
195-
if (data instanceof byte[]) {
196-
return new String((byte[]) data);
197-
}
198-
return data;
199-
}
200-
189+
case Types.LONGNVARCHAR:
201190
case Types.DATE:
202191
case Types.CLOB:
203192
default: {

‎src/main/java/com/microsoft/sqlserver/jdbc/dtv.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2063,7 +2063,8 @@ else if (type.isBinary()) {
20632063
// then do the conversion now so that the decision to use a "short" or "long"
20642064
// SSType (i.e. VARCHAR vs. TEXT/VARCHAR(max)) is based on the exact length of
20652065
// the MBCS value (in bytes).
2066-
else if (null != collation && (JDBCType.CHAR == type || JDBCType.VARCHAR == type
2066+
// If useBulkCopyForBatchInsert is true, conversion to byte array is not done due to performance
2067+
else if (!con.getUseBulkCopyForBatchInsert() && null != collation && (JDBCType.CHAR == type || JDBCType.VARCHAR == type
20672068
|| JDBCType.LONGVARCHAR == type || JDBCType.CLOB == type)) {
20682069
byte[] nativeEncoding = null;
20692070

‎src/test/java/com/microsoft/sqlserver/jdbc/preparedStatement/BatchExecutionWithBulkCopyTest.java

Lines changed: 230 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,13 @@
3030
import java.util.Calendar;
3131
import java.util.Random;
3232
import java.util.UUID;
33+
import java.util.logging.Handler;
34+
import java.util.logging.LogRecord;
35+
import java.util.logging.Logger;
3336

3437
import org.junit.jupiter.api.AfterAll;
35-
import org.junit.jupiter.api.BeforeEach;
3638
import org.junit.jupiter.api.BeforeAll;
39+
import org.junit.jupiter.api.BeforeEach;
3740
import org.junit.jupiter.api.Tag;
3841
import org.junit.jupiter.api.Test;
3942
import org.junit.platform.runner.JUnitPlatform;
@@ -1407,98 +1410,281 @@ private void getCreateTableTemporalSQL(String tableName) throws SQLException {
14071410
}
14081411
}
14091412

1413+
class FallbackWatcherLogHandler extends Handler implements AutoCloseable {
1414+
1415+
Logger stmtLogger = Logger.getLogger("com.microsoft.sqlserver.jdbc.internals.SQLServerStatement");
1416+
boolean gotFallbackMessage = false;
1417+
1418+
public FallbackWatcherLogHandler() {
1419+
stmtLogger.addHandler(this);
1420+
}
1421+
1422+
@Override
1423+
public void publish(LogRecord record) {
1424+
if (record.getMessage().contains("Falling back to the original implementation for Batch Insert.")) {
1425+
gotFallbackMessage = true;
1426+
}
1427+
}
1428+
1429+
@Override
1430+
public void flush() {}
1431+
1432+
@Override
1433+
public void close() throws SecurityException {
1434+
stmtLogger.removeHandler(this);
1435+
}
1436+
}
1437+
14101438
/**
1411-
* Test batch insert using bulk copy with string values when setSendStringParametersAsUnicode is true.
1439+
* Test string values using prepared statement using accented and unicode characters.
1440+
* This test covers all combinations of useBulkCopyForBatchInsert and sendStringParametersAsUnicode.
1441+
*
1442+
* @throws Exception
14121443
*/
14131444
@Test
1414-
public void testBulkInsertStringWhenSentAsUnicode() throws Exception {
1445+
public void testBulkInsertStringAllCombinations() throws Exception {
1446+
boolean[] bulkCopyOptions = { true, false };
1447+
boolean[] unicodeOptions = { true, false };
1448+
for (boolean useBulkCopy : bulkCopyOptions) {
1449+
for (boolean sendUnicode : unicodeOptions) {
1450+
runBulkInsertStringTest(useBulkCopy, sendUnicode);
1451+
runBulkInsertStringTestForceFallback(useBulkCopy, sendUnicode);
1452+
}
1453+
}
1454+
}
1455+
1456+
/**
1457+
* Test batch insert using accented and unicode characters.
1458+
*/
1459+
public void runBulkInsertStringTest(boolean useBulkCopy, boolean sendUnicode) throws Exception {
14151460
String insertSQL = "INSERT INTO " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString)
1416-
+ " (charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol) VALUES (?, ?, ?, ?, ?, ?)";
1461+
+ " (charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, longnvarcharCol1, "
1462+
+ "ncharCol2, nvarcharCol2, longnvarcharCol2) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)";
14171463

1418-
String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol FROM "
1464+
String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, "
1465+
+ "longnvarcharCol1, ncharCol2, nvarcharCol2, longnvarcharCol2 FROM "
14191466
+ AbstractSQLGenerator.escapeIdentifier(tableNameBulkString);
14201467

14211468
try (Connection connection = PrepUtil.getConnection(
1422-
connectionString + ";useBulkCopyForBatchInsert=true;sendStringParametersAsUnicode=true;");
1469+
connectionString + ";useBulkCopyForBatchInsert=" + useBulkCopy + ";sendStringParametersAsUnicode="
1470+
+ sendUnicode + ";");
14231471
SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
14241472
Statement stmt = (SQLServerStatement) connection.createStatement()) {
14251473

14261474
getCreateTableWithStringData();
14271475

1428-
pstmt.setString(1, "CHAR_VAL");
1429-
pstmt.setString(2, "VARCHAR_VALUE");
1430-
pstmt.setString(3, "LONGVARCHAR_VALUE_WITH_MORE_TEXT");
1431-
pstmt.setString(4, "NCHAR_VAL");
1432-
pstmt.setString(5, "NVARCHAR_VALUE");
1433-
pstmt.setString(6, "LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT");
1476+
String charValue = "Anaïs_Ni";
1477+
String varcharValue = "café";
1478+
String longVarcharValue = "Sørén Kierkégaard";
1479+
String ncharValue1 = "José Müll";
1480+
String nvarcharValue1 = "José Müller";
1481+
String longNvarcharValue1 = "François Saldaña";
1482+
String ncharValue2 = "Test1汉字😀";
1483+
String nvarcharValue2 = "汉字";
1484+
String longNvarcharValue2 = "日本語";
1485+
1486+
pstmt.setString(1, charValue);
1487+
pstmt.setString(2, varcharValue);
1488+
pstmt.setString(3, longVarcharValue);
1489+
pstmt.setString(4, ncharValue1);
1490+
pstmt.setString(5, nvarcharValue1);
1491+
pstmt.setString(6, longNvarcharValue1);
1492+
pstmt.setNString(7, ncharValue2);
1493+
pstmt.setNString(8, nvarcharValue2);
1494+
pstmt.setNString(9, longNvarcharValue2);
14341495
pstmt.addBatch();
14351496
pstmt.executeBatch();
14361497

14371498
// Validate inserted data
14381499
try (ResultSet rs = stmt.executeQuery(selectSQL)) {
14391500
assertTrue(rs.next(), "Expected at least one row in result set");
1440-
assertEquals("CHAR_VAL", rs.getString("charCol"));
1441-
assertEquals("VARCHAR_VALUE", rs.getString("varcharCol"));
1442-
assertEquals("LONGVARCHAR_VALUE_WITH_MORE_TEXT", rs.getString("longvarcharCol"));
1443-
assertEquals("NCHAR_VAL", rs.getString("ncharCol"));
1444-
assertEquals("NVARCHAR_VALUE", rs.getString("nvarcharCol"));
1445-
assertEquals("LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT", rs.getString("longnvarcharCol"));
1501+
assertEquals(charValue, rs.getString("charCol"));
1502+
assertEquals(varcharValue, rs.getString("varcharCol"));
1503+
assertEquals(longVarcharValue, rs.getString("longvarcharCol"));
1504+
assertEquals(ncharValue1, rs.getString("ncharCol1"));
1505+
assertEquals(nvarcharValue1, rs.getString("nvarcharCol1"));
1506+
assertEquals(longNvarcharValue1, rs.getString("longnvarcharCol1"));
1507+
assertEquals(ncharValue2, rs.getString("ncharCol2"));
1508+
assertEquals(nvarcharValue2, rs.getString("nvarcharCol2"));
1509+
assertEquals(longNvarcharValue2, rs.getString("longnvarcharCol2"));
14461510
assertFalse(rs.next());
14471511
}
14481512
}
14491513
}
14501514

14511515
/**
1452-
* Test batch insert using bulk copy with string values when setSendStringParametersAsUnicode is false.
1516+
* Test batch insert using an unsupported statement (falls back to batch mode) with accented and Unicode characters.
14531517
*/
1454-
@Test
1455-
public void testBulkInsertStringWhenNotSentAsUnicode() throws Exception {
1518+
public void runBulkInsertStringTestForceFallback(boolean useBulkCopy, boolean sendUnicode) throws Exception {
14561519
String insertSQL = "INSERT INTO " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString)
1457-
+ " (charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol) VALUES (?, ?, ?, ?, ?, ?)";
1520+
+ " (charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, longnvarcharCol1, "
1521+
+ "ncharCol2, nvarcharCol2, longnvarcharCol2) VALUES ('Anaïs_Ni', ?, ?, ?, ?, ?, ?, ?, ?)";
14581522

1459-
String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol FROM "
1523+
String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, "
1524+
+ "longnvarcharCol1, ncharCol2, nvarcharCol2, longnvarcharCol2 FROM "
14601525
+ AbstractSQLGenerator.escapeIdentifier(tableNameBulkString);
14611526

1462-
try (Connection connection = PrepUtil.getConnection(
1463-
connectionString + ";useBulkCopyForBatchInsert=true;sendStringParametersAsUnicode=false;");
1527+
try (Connection connection = PrepUtil.getConnection(connectionString + ";useBulkCopyForBatchInsert="
1528+
+ useBulkCopy + ";sendStringParametersAsUnicode=" + sendUnicode + ";");
14641529
SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
14651530
Statement stmt = (SQLServerStatement) connection.createStatement()) {
14661531

14671532
getCreateTableWithStringData();
14681533

1469-
pstmt.setString(1, "CHAR_VAL");
1470-
pstmt.setString(2, "VARCHAR_VALUE");
1471-
pstmt.setString(3, "LONGVARCHAR_VALUE_WITH_MORE_TEXT");
1472-
pstmt.setString(4, "NCHAR_VAL");
1473-
pstmt.setString(5, "NVARCHAR_VALUE");
1474-
pstmt.setString(6, "LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT");
1534+
String charValue = "Anaïs_Ni";
1535+
String varcharValue = "café";
1536+
String longVarcharValue = "Sørén Kierkégaard";
1537+
String ncharValue1 = "José Müll";
1538+
String nvarcharValue1 = "José Müller";
1539+
String longNvarcharValue1 = "François Saldaña";
1540+
String ncharValue2 = "Test1汉字😀";
1541+
String nvarcharValue2 = "汉字";
1542+
String longNvarcharValue2 = "日本語";
1543+
1544+
pstmt.setString(1, varcharValue);
1545+
pstmt.setString(2, longVarcharValue);
1546+
pstmt.setString(3, ncharValue1);
1547+
pstmt.setString(4, nvarcharValue1);
1548+
pstmt.setString(5, longNvarcharValue1);
1549+
pstmt.setNString(6, ncharValue2);
1550+
pstmt.setNString(7, nvarcharValue2);
1551+
pstmt.setNString(8, longNvarcharValue2);
1552+
pstmt.addBatch();
1553+
1554+
try (FallbackWatcherLogHandler handler = new FallbackWatcherLogHandler()) {
1555+
pstmt.executeBatch();
1556+
if (useBulkCopy) {
1557+
assertTrue(handler.gotFallbackMessage);
1558+
}
1559+
}
1560+
1561+
// Validate inserted data
1562+
try (ResultSet rs = stmt.executeQuery(selectSQL)) {
1563+
assertTrue(rs.next(), "Expected at least one row in result set");
1564+
assertEquals(charValue, rs.getString("charCol"));
1565+
assertEquals(varcharValue, rs.getString("varcharCol"));
1566+
assertEquals(longVarcharValue, rs.getString("longvarcharCol"));
1567+
assertEquals(ncharValue1, rs.getString("ncharCol1"));
1568+
assertEquals(nvarcharValue1, rs.getString("nvarcharCol1"));
1569+
assertEquals(longNvarcharValue1, rs.getString("longnvarcharCol1"));
1570+
assertEquals(ncharValue2, rs.getString("ncharCol2"));
1571+
assertEquals(nvarcharValue2, rs.getString("nvarcharCol2"));
1572+
assertEquals(longNvarcharValue2, rs.getString("longnvarcharCol2"));
1573+
assertFalse(rs.next());
1574+
}
1575+
}
1576+
}
1577+
1578+
@Test
1579+
public void testIssue2669Repro() throws Exception {
1580+
// Original repro
1581+
testIssue2669Variation(false, true);
1582+
// Variations
1583+
testIssue2669Variation(false, false);
1584+
testIssue2669Variation(true, true);
1585+
testIssue2669Variation(true, false);
1586+
1587+
// Test the same combos except falling back to batch insert
1588+
testIssue2669VariationForceFallback(false, true);
1589+
testIssue2669VariationForceFallback(false, false);
1590+
testIssue2669VariationForceFallback(true, true);
1591+
testIssue2669VariationForceFallback(true, false);
1592+
}
1593+
1594+
public void testIssue2669Variation(boolean sendStringsAsUnicode,
1595+
boolean useBulkCopyForBatchInsert) throws Exception {
1596+
String statesTable = AbstractSQLGenerator.escapeIdentifier(RandomUtil.getIdentifier("states"));
1597+
try (Statement stmt = connection.createStatement()) {
1598+
TestUtils.dropTableIfExists(statesTable, stmt);
1599+
String createTableSQL = "CREATE TABLE " + statesTable + " (" + "StateCode nvarchar(50) NOT NULL " + ")";
1600+
1601+
stmt.execute(createTableSQL);
1602+
}
1603+
1604+
String insertSQL = "INSERT INTO " + statesTable + " (StateCode) VALUES (?)";
1605+
1606+
String selectSQL = "SELECT StateCode FROM " + statesTable;
1607+
1608+
try (Connection connection = PrepUtil.getConnection(connectionString + ";useBulkCopyForBatchInsert="
1609+
+ useBulkCopyForBatchInsert + ";sendStringParametersAsUnicode=" + sendStringsAsUnicode + ";");
1610+
SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
1611+
Statement stmt = (SQLServerStatement) connection.createStatement()) {
1612+
1613+
pstmt.setString(1, "OH");
14751614
pstmt.addBatch();
14761615
pstmt.executeBatch();
14771616

14781617
// Validate inserted data
14791618
try (ResultSet rs = stmt.executeQuery(selectSQL)) {
14801619
assertTrue(rs.next(), "Expected at least one row in result set");
1481-
assertEquals("CHAR_VAL", rs.getString("charCol"));
1482-
assertEquals("VARCHAR_VALUE", rs.getString("varcharCol"));
1483-
assertEquals("LONGVARCHAR_VALUE_WITH_MORE_TEXT", rs.getString("longvarcharCol"));
1484-
assertEquals("NCHAR_VAL", rs.getString("ncharCol"));
1485-
assertEquals("NVARCHAR_VALUE", rs.getString("nvarcharCol"));
1486-
assertEquals("LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT", rs.getString("longnvarcharCol"));
1620+
assertEquals("OH", rs.getString("StateCode"));
14871621
assertFalse(rs.next());
14881622
}
1623+
} finally {
1624+
try (Statement stmt = connection.createStatement()) {
1625+
TestUtils.dropTableIfExists(statesTable, stmt);
1626+
}
1627+
}
1628+
}
1629+
1630+
public void testIssue2669VariationForceFallback(boolean sendStringsAsUnicode,
1631+
boolean useBulkCopyForBatchInsert) throws SQLException {
1632+
String statesTable = AbstractSQLGenerator.escapeIdentifier(RandomUtil.getIdentifier("states"));
1633+
try (Statement stmt = connection.createStatement()) {
1634+
TestUtils.dropTableIfExists(statesTable, stmt);
1635+
String createTableSQL = "CREATE TABLE " + statesTable
1636+
+ " (StateCode nvarchar(50), StateCode2 nvarchar(50) NOT NULL " + ")";
1637+
1638+
stmt.execute(createTableSQL);
1639+
}
1640+
1641+
// Use an INSERT that forces fall back to plain batch insert
1642+
String insertSQL = "INSERT INTO " + statesTable + " (StateCode, StateCode2) VALUES ('NA', ?)";
1643+
1644+
String selectSQL = "SELECT StateCode, StateCode2 FROM " + statesTable;
1645+
1646+
try (Connection connection = PrepUtil.getConnection(connectionString + ";useBulkCopyForBatchInsert="
1647+
+ useBulkCopyForBatchInsert + ";sendStringParametersAsUnicode=" + sendStringsAsUnicode + ";");
1648+
SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
1649+
Statement stmt = (SQLServerStatement) connection.createStatement()) {
1650+
1651+
pstmt.setString(1, "OH");
1652+
pstmt.addBatch();
1653+
1654+
try (FallbackWatcherLogHandler handler = new FallbackWatcherLogHandler()) {
1655+
pstmt.executeBatch();
1656+
if (useBulkCopyForBatchInsert) {
1657+
assertTrue(handler.gotFallbackMessage);
1658+
}
1659+
}
1660+
1661+
// Validate inserted data
1662+
try (ResultSet rs = stmt.executeQuery(selectSQL)) {
1663+
assertTrue(rs.next(), "Expected at least one row in result set");
1664+
assertEquals("NA", rs.getString("StateCode"));
1665+
assertEquals("OH", rs.getString("StateCode2"));
1666+
assertFalse(rs.next());
1667+
}
1668+
} finally {
1669+
try (Statement stmt = connection.createStatement()) {
1670+
TestUtils.dropTableIfExists(statesTable, stmt);
1671+
}
14891672
}
14901673
}
14911674

14921675
private void getCreateTableWithStringData() throws SQLException {
14931676
try (Statement stmt = connection.createStatement()) {
14941677
TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(tableNameBulkString), stmt);
14951678
String createTableSQL = "CREATE TABLE " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString) + " (" +
1496-
"charCol CHAR(8) NOT NULL, " +
1497-
"varcharCol VARCHAR(50) NOT NULL, " +
1498-
"longvarcharCol VARCHAR(MAX) NOT NULL, " +
1499-
"ncharCol NCHAR(9) NOT NULL, " +
1500-
"nvarcharCol NVARCHAR(50) NOT NULL, " +
1501-
"longnvarcharCol NVARCHAR(MAX) NOT NULL" + ")";
1679+
"charCol CHAR(8), " +
1680+
"varcharCol VARCHAR(50), " +
1681+
"longvarcharCol VARCHAR(MAX), " +
1682+
"ncharCol1 NCHAR(9), " +
1683+
"nvarcharCol1 NVARCHAR(50), " +
1684+
"longnvarcharCol1 NVARCHAR(MAX), " +
1685+
"ncharCol2 NCHAR(9), " +
1686+
"nvarcharCol2 NVARCHAR(50), " +
1687+
"longnvarcharCol2 NVARCHAR(MAX)" + ")";
15021688

15031689
stmt.execute(createTableSQL);
15041690
}
@@ -1534,6 +1720,7 @@ public static void terminateVariation() throws SQLException {
15341720
TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(doubleQuoteTableName), stmt);
15351721
TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(schemaTableName), stmt);
15361722
TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(tableNameBulkString), stmt);
1723+
TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(tableNameBulkComputedCols), stmt);
15371724
}
15381725
}
15391726
}

0 commit comments

Comments
 (0)