Skip to content

Addressed performance issue on Bulk Copy API with batch insert #2735

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Aug 14, 2025
Merged

Addressed performance issue on Bulk Copy API with batch insert #2735

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f32cb53
Addressed performance issue on Bulk Copy API with batch insert
muskan124947 Aug 11, 2025
81e8c4d
Added test to verify fallback to normal execution for bulk copy api
muskan124947 Aug 12, 2025
5959f2f
Addressed comment
muskan124947 Aug 12, 2025
80d67c9
Update BatchExecutionWithBulkCopyTest.java
muskan124947 Aug 12, 2025
3fc45fc
Few more test combinations
David-Engel Aug 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 1 addition & 12 deletions src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkBatchInsertRecord.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,18 +186,7 @@ else if (dateTimeFormatter != null)
case Types.LONGVARCHAR:
case Types.NCHAR:
case Types.NVARCHAR:
case Types.LONGNVARCHAR: {
/*
* If string data comes in as a byte array through setString (and sendStringParametersAsUnicode = false)
* through Bulk Copy for Batch Insert API, convert the byte array to a string.
* If the data is already a string, return it as is.
*/
if (data instanceof byte[]) {
return new String((byte[]) data);
}
return data;
}

case Types.LONGNVARCHAR:
case Types.DATE:
case Types.CLOB:
default: {
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/com/microsoft/sqlserver/jdbc/dtv.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2063,7 +2063,8 @@ else if (type.isBinary()) {
// then do the conversion now so that the decision to use a "short" or "long"
// SSType (i.e. VARCHAR vs. TEXT/VARCHAR(max)) is based on the exact length of
// the MBCS value (in bytes).
else if (null != collation && (JDBCType.CHAR == type || JDBCType.VARCHAR == type
// If useBulkCopyForBatchInsert is true, conversion to byte array is not done due to performance
else if (!con.getUseBulkCopyForBatchInsert() && null != collation && (JDBCType.CHAR == type || JDBCType.VARCHAR == type
|| JDBCType.LONGVARCHAR == type || JDBCType.CLOB == type)) {
byte[] nativeEncoding = null;

Expand Down
273 changes: 230 additions & 43 deletions ...t/java/com/microsoft/sqlserver/jdbc/preparedStatement/BatchExecutionWithBulkCopyTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,13 @@
import java.util.Calendar;
import java.util.Random;
import java.util.UUID;
import java.util.logging.Handler;
import java.util.logging.LogRecord;
import java.util.logging.Logger;

import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.platform.runner.JUnitPlatform;
Expand Down Expand Up @@ -1407,98 +1410,281 @@ private void getCreateTableTemporalSQL(String tableName) throws SQLException {
}
}

class FallbackWatcherLogHandler extends Handler implements AutoCloseable {

Logger stmtLogger = Logger.getLogger("com.microsoft.sqlserver.jdbc.internals.SQLServerStatement");
boolean gotFallbackMessage = false;

public FallbackWatcherLogHandler() {
stmtLogger.addHandler(this);
}

@Override
public void publish(LogRecord record) {
if (record.getMessage().contains("Falling back to the original implementation for Batch Insert.")) {
gotFallbackMessage = true;
}
}

@Override
public void flush() {}

@Override
public void close() throws SecurityException {
stmtLogger.removeHandler(this);
}
}

/**
* Test batch insert using bulk copy with string values when setSendStringParametersAsUnicode is true.
* Test string values using prepared statement using accented and unicode characters.
* This test covers all combinations of useBulkCopyForBatchInsert and sendStringParametersAsUnicode.
*
* @throws Exception
*/
@Test
public void testBulkInsertStringWhenSentAsUnicode() throws Exception {
public void testBulkInsertStringAllCombinations() throws Exception {
boolean[] bulkCopyOptions = { true, false };
boolean[] unicodeOptions = { true, false };
for (boolean useBulkCopy : bulkCopyOptions) {
for (boolean sendUnicode : unicodeOptions) {
runBulkInsertStringTest(useBulkCopy, sendUnicode);
runBulkInsertStringTestForceFallback(useBulkCopy, sendUnicode);
}
}
}

/**
* Test batch insert using accented and unicode characters.
*/
public void runBulkInsertStringTest(boolean useBulkCopy, boolean sendUnicode) throws Exception {
String insertSQL = "INSERT INTO " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString)
+ " (charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol) VALUES (?, ?, ?, ?, ?, ?)";
+ " (charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, longnvarcharCol1, "
+ "ncharCol2, nvarcharCol2, longnvarcharCol2) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)";

String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol FROM "
String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, "
+ "longnvarcharCol1, ncharCol2, nvarcharCol2, longnvarcharCol2 FROM "
+ AbstractSQLGenerator.escapeIdentifier(tableNameBulkString);

try (Connection connection = PrepUtil.getConnection(
connectionString + ";useBulkCopyForBatchInsert=true;sendStringParametersAsUnicode=true;");
connectionString + ";useBulkCopyForBatchInsert=" + useBulkCopy + ";sendStringParametersAsUnicode="
+ sendUnicode + ";");
SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
Statement stmt = (SQLServerStatement) connection.createStatement()) {

getCreateTableWithStringData();

pstmt.setString(1, "CHAR_VAL");
pstmt.setString(2, "VARCHAR_VALUE");
pstmt.setString(3, "LONGVARCHAR_VALUE_WITH_MORE_TEXT");
pstmt.setString(4, "NCHAR_VAL");
pstmt.setString(5, "NVARCHAR_VALUE");
pstmt.setString(6, "LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT");
String charValue = "Anaïs_Ni";
String varcharValue = "café";
String longVarcharValue = "Sørén Kierkégaard";
String ncharValue1 = "José Müll";
String nvarcharValue1 = "José Müller";
String longNvarcharValue1 = "François Saldaña";
String ncharValue2 = "Test1汉字😀";
String nvarcharValue2 = "汉字";
String longNvarcharValue2 = "日本語";

pstmt.setString(1, charValue);
pstmt.setString(2, varcharValue);
pstmt.setString(3, longVarcharValue);
pstmt.setString(4, ncharValue1);
pstmt.setString(5, nvarcharValue1);
pstmt.setString(6, longNvarcharValue1);
pstmt.setNString(7, ncharValue2);
pstmt.setNString(8, nvarcharValue2);
pstmt.setNString(9, longNvarcharValue2);
pstmt.addBatch();
pstmt.executeBatch();

// Validate inserted data
try (ResultSet rs = stmt.executeQuery(selectSQL)) {
assertTrue(rs.next(), "Expected at least one row in result set");
assertEquals("CHAR_VAL", rs.getString("charCol"));
assertEquals("VARCHAR_VALUE", rs.getString("varcharCol"));
assertEquals("LONGVARCHAR_VALUE_WITH_MORE_TEXT", rs.getString("longvarcharCol"));
assertEquals("NCHAR_VAL", rs.getString("ncharCol"));
assertEquals("NVARCHAR_VALUE", rs.getString("nvarcharCol"));
assertEquals("LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT", rs.getString("longnvarcharCol"));
assertEquals(charValue, rs.getString("charCol"));
assertEquals(varcharValue, rs.getString("varcharCol"));
assertEquals(longVarcharValue, rs.getString("longvarcharCol"));
assertEquals(ncharValue1, rs.getString("ncharCol1"));
assertEquals(nvarcharValue1, rs.getString("nvarcharCol1"));
assertEquals(longNvarcharValue1, rs.getString("longnvarcharCol1"));
assertEquals(ncharValue2, rs.getString("ncharCol2"));
assertEquals(nvarcharValue2, rs.getString("nvarcharCol2"));
assertEquals(longNvarcharValue2, rs.getString("longnvarcharCol2"));
assertFalse(rs.next());
}
}
}

/**
* Test batch insert using bulk copy with string values when setSendStringParametersAsUnicode is false.
* Test batch insert using an unsupported statement (falls back to batch mode) with accented and Unicode characters.
*/
@Test
public void testBulkInsertStringWhenNotSentAsUnicode() throws Exception {
public void runBulkInsertStringTestForceFallback(boolean useBulkCopy, boolean sendUnicode) throws Exception {
String insertSQL = "INSERT INTO " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString)
+ " (charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol) VALUES (?, ?, ?, ?, ?, ?)";
+ " (charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, longnvarcharCol1, "
+ "ncharCol2, nvarcharCol2, longnvarcharCol2) VALUES ('Anaïs_Ni', ?, ?, ?, ?, ?, ?, ?, ?)";

String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol FROM "
String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, "
+ "longnvarcharCol1, ncharCol2, nvarcharCol2, longnvarcharCol2 FROM "
+ AbstractSQLGenerator.escapeIdentifier(tableNameBulkString);

try (Connection connection = PrepUtil.getConnection(
connectionString + ";useBulkCopyForBatchInsert=true;sendStringParametersAsUnicode=false;");
try (Connection connection = PrepUtil.getConnection(connectionString + ";useBulkCopyForBatchInsert="
+ useBulkCopy + ";sendStringParametersAsUnicode=" + sendUnicode + ";");
SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
Statement stmt = (SQLServerStatement) connection.createStatement()) {

getCreateTableWithStringData();

pstmt.setString(1, "CHAR_VAL");
pstmt.setString(2, "VARCHAR_VALUE");
pstmt.setString(3, "LONGVARCHAR_VALUE_WITH_MORE_TEXT");
pstmt.setString(4, "NCHAR_VAL");
pstmt.setString(5, "NVARCHAR_VALUE");
pstmt.setString(6, "LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT");
String charValue = "Anaïs_Ni";
String varcharValue = "café";
String longVarcharValue = "Sørén Kierkégaard";
String ncharValue1 = "José Müll";
String nvarcharValue1 = "José Müller";
String longNvarcharValue1 = "François Saldaña";
String ncharValue2 = "Test1汉字😀";
String nvarcharValue2 = "汉字";
String longNvarcharValue2 = "日本語";

pstmt.setString(1, varcharValue);
pstmt.setString(2, longVarcharValue);
pstmt.setString(3, ncharValue1);
pstmt.setString(4, nvarcharValue1);
pstmt.setString(5, longNvarcharValue1);
pstmt.setNString(6, ncharValue2);
pstmt.setNString(7, nvarcharValue2);
pstmt.setNString(8, longNvarcharValue2);
pstmt.addBatch();

try (FallbackWatcherLogHandler handler = new FallbackWatcherLogHandler()) {
pstmt.executeBatch();
if (useBulkCopy) {
assertTrue(handler.gotFallbackMessage);
}
}

// Validate inserted data
try (ResultSet rs = stmt.executeQuery(selectSQL)) {
assertTrue(rs.next(), "Expected at least one row in result set");
assertEquals(charValue, rs.getString("charCol"));
assertEquals(varcharValue, rs.getString("varcharCol"));
assertEquals(longVarcharValue, rs.getString("longvarcharCol"));
assertEquals(ncharValue1, rs.getString("ncharCol1"));
assertEquals(nvarcharValue1, rs.getString("nvarcharCol1"));
assertEquals(longNvarcharValue1, rs.getString("longnvarcharCol1"));
assertEquals(ncharValue2, rs.getString("ncharCol2"));
assertEquals(nvarcharValue2, rs.getString("nvarcharCol2"));
assertEquals(longNvarcharValue2, rs.getString("longnvarcharCol2"));
assertFalse(rs.next());
}
}
}

@Test
public void testIssue2669Repro() throws Exception {
// Original repro
testIssue2669Variation(false, true);
// Variations
testIssue2669Variation(false, false);
testIssue2669Variation(true, true);
testIssue2669Variation(true, false);

// Test the same combos except falling back to batch insert
testIssue2669VariationForceFallback(false, true);
testIssue2669VariationForceFallback(false, false);
testIssue2669VariationForceFallback(true, true);
testIssue2669VariationForceFallback(true, false);
}

public void testIssue2669Variation(boolean sendStringsAsUnicode,
boolean useBulkCopyForBatchInsert) throws Exception {
String statesTable = AbstractSQLGenerator.escapeIdentifier(RandomUtil.getIdentifier("states"));
try (Statement stmt = connection.createStatement()) {
TestUtils.dropTableIfExists(statesTable, stmt);
String createTableSQL = "CREATE TABLE " + statesTable + " (" + "StateCode nvarchar(50) NOT NULL " + ")";

stmt.execute(createTableSQL);
}

String insertSQL = "INSERT INTO " + statesTable + " (StateCode) VALUES (?)";

String selectSQL = "SELECT StateCode FROM " + statesTable;

try (Connection connection = PrepUtil.getConnection(connectionString + ";useBulkCopyForBatchInsert="
+ useBulkCopyForBatchInsert + ";sendStringParametersAsUnicode=" + sendStringsAsUnicode + ";");
SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
Statement stmt = (SQLServerStatement) connection.createStatement()) {

pstmt.setString(1, "OH");
pstmt.addBatch();
pstmt.executeBatch();

// Validate inserted data
try (ResultSet rs = stmt.executeQuery(selectSQL)) {
assertTrue(rs.next(), "Expected at least one row in result set");
assertEquals("CHAR_VAL", rs.getString("charCol"));
assertEquals("VARCHAR_VALUE", rs.getString("varcharCol"));
assertEquals("LONGVARCHAR_VALUE_WITH_MORE_TEXT", rs.getString("longvarcharCol"));
assertEquals("NCHAR_VAL", rs.getString("ncharCol"));
assertEquals("NVARCHAR_VALUE", rs.getString("nvarcharCol"));
assertEquals("LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT", rs.getString("longnvarcharCol"));
assertEquals("OH", rs.getString("StateCode"));
assertFalse(rs.next());
}
} finally {
try (Statement stmt = connection.createStatement()) {
TestUtils.dropTableIfExists(statesTable, stmt);
}
}
}

public void testIssue2669VariationForceFallback(boolean sendStringsAsUnicode,
boolean useBulkCopyForBatchInsert) throws SQLException {
String statesTable = AbstractSQLGenerator.escapeIdentifier(RandomUtil.getIdentifier("states"));
try (Statement stmt = connection.createStatement()) {
TestUtils.dropTableIfExists(statesTable, stmt);
String createTableSQL = "CREATE TABLE " + statesTable
+ " (StateCode nvarchar(50), StateCode2 nvarchar(50) NOT NULL " + ")";

stmt.execute(createTableSQL);
}

// Use an INSERT that forces fall back to plain batch insert
String insertSQL = "INSERT INTO " + statesTable + " (StateCode, StateCode2) VALUES ('NA', ?)";

String selectSQL = "SELECT StateCode, StateCode2 FROM " + statesTable;

try (Connection connection = PrepUtil.getConnection(connectionString + ";useBulkCopyForBatchInsert="
+ useBulkCopyForBatchInsert + ";sendStringParametersAsUnicode=" + sendStringsAsUnicode + ";");
SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
Statement stmt = (SQLServerStatement) connection.createStatement()) {

pstmt.setString(1, "OH");
pstmt.addBatch();

try (FallbackWatcherLogHandler handler = new FallbackWatcherLogHandler()) {
pstmt.executeBatch();
if (useBulkCopyForBatchInsert) {
assertTrue(handler.gotFallbackMessage);
}
}

// Validate inserted data
try (ResultSet rs = stmt.executeQuery(selectSQL)) {
assertTrue(rs.next(), "Expected at least one row in result set");
assertEquals("NA", rs.getString("StateCode"));
assertEquals("OH", rs.getString("StateCode2"));
assertFalse(rs.next());
}
} finally {
try (Statement stmt = connection.createStatement()) {
TestUtils.dropTableIfExists(statesTable, stmt);
}
}
}

private void getCreateTableWithStringData() throws SQLException {
try (Statement stmt = connection.createStatement()) {
TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(tableNameBulkString), stmt);
String createTableSQL = "CREATE TABLE " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString) + " (" +
"charCol CHAR(8) NOT NULL, " +
"varcharCol VARCHAR(50) NOT NULL, " +
"longvarcharCol VARCHAR(MAX) NOT NULL, " +
"ncharCol NCHAR(9) NOT NULL, " +
"nvarcharCol NVARCHAR(50) NOT NULL, " +
"longnvarcharCol NVARCHAR(MAX) NOT NULL" + ")";
"charCol CHAR(8), " +
"varcharCol VARCHAR(50), " +
"longvarcharCol VARCHAR(MAX), " +
"ncharCol1 NCHAR(9), " +
"nvarcharCol1 NVARCHAR(50), " +
"longnvarcharCol1 NVARCHAR(MAX), " +
"ncharCol2 NCHAR(9), " +
"nvarcharCol2 NVARCHAR(50), " +
"longnvarcharCol2 NVARCHAR(MAX)" + ")";

stmt.execute(createTableSQL);
}
Expand Down Expand Up @@ -1534,6 +1720,7 @@ public static void terminateVariation() throws SQLException {
TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(doubleQuoteTableName), stmt);
TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(schemaTableName), stmt);
TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(tableNameBulkString), stmt);
TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(tableNameBulkComputedCols), stmt);
}
}
}
Loading