Skip to content

Addressed performance issue on Bulk Copy API with batch insert #2735

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Aug 14, 2025
Merged

Addressed performance issue on Bulk Copy API with batch insert #2735

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f32cb53
Addressed performance issue on Bulk Copy API with batch insert
muskan124947 Aug 11, 2025
81e8c4d
Added test to verify fallback to normal execution for bulk copy api
muskan124947 Aug 12, 2025
5959f2f
Addressed comment
muskan124947 Aug 12, 2025
80d67c9
Update BatchExecutionWithBulkCopyTest.java
muskan124947 Aug 12, 2025
3fc45fc
Few more test combinations
David-Engel Aug 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 1 addition & 12 deletions src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkBatchInsertRecord.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,18 +186,7 @@ else if (dateTimeFormatter != null)
case Types.LONGVARCHAR:
case Types.NCHAR:
case Types.NVARCHAR:
case Types.LONGNVARCHAR: {
/*
* If string data comes in as a byte array through setString (and sendStringParametersAsUnicode = false)
* through Bulk Copy for Batch Insert API, convert the byte array to a string.
* If the data is already a string, return it as is.
*/
if (data instanceof byte[]) {
return new String((byte[]) data);
}
return data;
}

case Types.LONGNVARCHAR:
case Types.DATE:
case Types.CLOB:
default: {
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/com/microsoft/sqlserver/jdbc/dtv.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2063,7 +2063,8 @@ else if (type.isBinary()) {
// then do the conversion now so that the decision to use a "short" or "long"
// SSType (i.e. VARCHAR vs. TEXT/VARCHAR(max)) is based on the exact length of
// the MBCS value (in bytes).
else if (null != collation && (JDBCType.CHAR == type || JDBCType.VARCHAR == type
// If useBulkCopyForBatchInsert is true, conversion to byte array is not done due to performance
else if (!con.getUseBulkCopyForBatchInsert() && null != collation && (JDBCType.CHAR == type || JDBCType.VARCHAR == type
|| JDBCType.LONGVARCHAR == type || JDBCType.CLOB == type)) {
byte[] nativeEncoding = null;

Expand Down
146 changes: 96 additions & 50 deletions ...t/java/com/microsoft/sqlserver/jdbc/preparedStatement/BatchExecutionWithBulkCopyTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.util.UUID;

import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
Expand Down Expand Up @@ -1408,82 +1409,124 @@ private void getCreateTableTemporalSQL(String tableName) throws SQLException {
}

/**
* Test batch insert using bulk copy with string values when setSendStringParametersAsUnicode is true.
* Test insert-select fallback to normal execution for bulk copy API
*/
@Test
public void testBulkInsertStringWhenSentAsUnicode() throws Exception {
String insertSQL = "INSERT INTO " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString)
+ " (charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol) VALUES (?, ?, ?, ?, ?, ?)";
public void testInsertSelectFallbackToNormalExecution() throws Exception {
String tableNameSource = AbstractSQLGenerator.escapeIdentifier("SourceTable");
String tableNameDestination = AbstractSQLGenerator.escapeIdentifier("DestinationTable");

String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol FROM "
+ AbstractSQLGenerator.escapeIdentifier(tableNameBulkString);
String connectStringUrl = connectionString
+ ";useBulkCopyForBatchInsert=true;sendStringParametersAsUnicode=false;";

try (Connection connection = PrepUtil.getConnection(
connectionString + ";useBulkCopyForBatchInsert=true;sendStringParametersAsUnicode=true;");
SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
Statement stmt = (SQLServerStatement) connection.createStatement()) {
try (Connection connection = PrepUtil.getConnection(connectStringUrl);
Statement stmt = connection.createStatement()) {

getCreateTableWithStringData();
TestUtils.dropTableIfExists(tableNameSource, stmt);
String createSourceTableSQL = "CREATE TABLE " + tableNameSource + " (id INT, value VARCHAR(50))";
stmt.execute(createSourceTableSQL);

pstmt.setString(1, "CHAR_VAL");
pstmt.setString(2, "VARCHAR_VALUE");
pstmt.setString(3, "LONGVARCHAR_VALUE_WITH_MORE_TEXT");
pstmt.setString(4, "NCHAR_VAL");
pstmt.setString(5, "NVARCHAR_VALUE");
pstmt.setString(6, "LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT");
pstmt.addBatch();
pstmt.executeBatch();
String insertSourceDataSQL = "INSERT INTO " + tableNameSource + " VALUES (1, 'TestValue1'), (2, 'TestValue2')";
stmt.execute(insertSourceDataSQL);

// Validate inserted data
TestUtils.dropTableIfExists(tableNameDestination, stmt);
String createDestinationTableSQL = "CREATE TABLE " + tableNameDestination + " (id INT, value VARCHAR(50))";
stmt.execute(createDestinationTableSQL);

// Attempt unsupported INSERT-SELECT query for bulk copy api
String insertSelectSQL = "INSERT INTO " + tableNameDestination + " SELECT * FROM " + tableNameSource
+ " WHERE value = ?";

try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection
.prepareStatement(insertSelectSQL)) {
pstmt.setString(1, "TestValue1");
pstmt.addBatch();
pstmt.executeBatch(); // This should fall back to normal execution flow
}

// Validate inserted data in destination table
String selectSQL = "SELECT * FROM " + tableNameDestination;
try (ResultSet rs = stmt.executeQuery(selectSQL)) {
assertTrue(rs.next(), "Expected at least one row in result set");
assertEquals("CHAR_VAL", rs.getString("charCol"));
assertEquals("VARCHAR_VALUE", rs.getString("varcharCol"));
assertEquals("LONGVARCHAR_VALUE_WITH_MORE_TEXT", rs.getString("longvarcharCol"));
assertEquals("NCHAR_VAL", rs.getString("ncharCol"));
assertEquals("NVARCHAR_VALUE", rs.getString("nvarcharCol"));
assertEquals("LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT", rs.getString("longnvarcharCol"));
assertFalse(rs.next());
assertEquals(1, rs.getInt("id"));
assertEquals("TestValue1", rs.getString("value"));

Assertions.assertFalse(rs.next(), "No more rows expected");
}
}
}

/**
* Test batch insert using bulk copy with string values when setSendStringParametersAsUnicode is false.
* Test string values using prepared statement using accented and unicode characters.
* This test covers all combinations of useBulkCopyForBatchInsert and sendStringParametersAsUnicode.
*
* @throws Exception
*/
@Test
public void testBulkInsertStringWhenNotSentAsUnicode() throws Exception {
public void testBulkInsertStringAllCombinations() throws Exception {
boolean[] bulkCopyOptions = { true, false };
boolean[] unicodeOptions = { true, false };
for (boolean useBulkCopy : bulkCopyOptions) {
for (boolean sendUnicode : unicodeOptions) {
runBulkInsertStringTest(useBulkCopy, sendUnicode);
}
}
}

/**
* Test batch insert using accented and unicode characters.
*/
public void runBulkInsertStringTest(boolean useBulkCopy, boolean sendUnicode) throws Exception {
String insertSQL = "INSERT INTO " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString)
+ " (charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol) VALUES (?, ?, ?, ?, ?, ?)";
+ " (charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, longnvarcharCol1, "
+ "ncharCol2, nvarcharCol2, longnvarcharCol2) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)";

String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol FROM "
String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, "
+ "longnvarcharCol1, ncharCol2, nvarcharCol2, longnvarcharCol2 FROM "
+ AbstractSQLGenerator.escapeIdentifier(tableNameBulkString);

try (Connection connection = PrepUtil.getConnection(
connectionString + ";useBulkCopyForBatchInsert=true;sendStringParametersAsUnicode=false;");
connectionString + ";useBulkCopyForBatchInsert=" + useBulkCopy + ";sendStringParametersAsUnicode="
+ sendUnicode + ";");
SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
Statement stmt = (SQLServerStatement) connection.createStatement()) {

getCreateTableWithStringData();

pstmt.setString(1, "CHAR_VAL");
pstmt.setString(2, "VARCHAR_VALUE");
pstmt.setString(3, "LONGVARCHAR_VALUE_WITH_MORE_TEXT");
pstmt.setString(4, "NCHAR_VAL");
pstmt.setString(5, "NVARCHAR_VALUE");
pstmt.setString(6, "LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT");
String charValue = "Anaïs_Ni";
String varcharValue = "café";
String longVarcharValue = "Sørén Kierkégaard";
String ncharValue1 = "José Müll";
String nvarcharValue1 = "José Müller";
String longNvarcharValue1 = "François Saldaña";
String ncharValue2 = "Test1汉字😀";
String nvarcharValue2 = "汉字";
String longNvarcharValue2 = "日本語";

pstmt.setString(1, charValue);
pstmt.setString(2, varcharValue);
pstmt.setString(3, longVarcharValue);
pstmt.setString(4, ncharValue1);
pstmt.setString(5, nvarcharValue1);
pstmt.setString(6, longNvarcharValue1);
pstmt.setNString(7, ncharValue2);
pstmt.setNString(8, nvarcharValue2);
pstmt.setNString(9, longNvarcharValue2);
pstmt.addBatch();
pstmt.executeBatch();

// Validate inserted data
try (ResultSet rs = stmt.executeQuery(selectSQL)) {
assertTrue(rs.next(), "Expected at least one row in result set");
assertEquals("CHAR_VAL", rs.getString("charCol"));
assertEquals("VARCHAR_VALUE", rs.getString("varcharCol"));
assertEquals("LONGVARCHAR_VALUE_WITH_MORE_TEXT", rs.getString("longvarcharCol"));
assertEquals("NCHAR_VAL", rs.getString("ncharCol"));
assertEquals("NVARCHAR_VALUE", rs.getString("nvarcharCol"));
assertEquals("LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT", rs.getString("longnvarcharCol"));
assertEquals(charValue, rs.getString("charCol"));
assertEquals(varcharValue, rs.getString("varcharCol"));
assertEquals(longVarcharValue, rs.getString("longvarcharCol"));
assertEquals(ncharValue1, rs.getString("ncharCol1"));
assertEquals(nvarcharValue1, rs.getString("nvarcharCol1"));
assertEquals(longNvarcharValue1, rs.getString("longnvarcharCol1"));
assertEquals(ncharValue2, rs.getString("ncharCol2"));
assertEquals(nvarcharValue2, rs.getString("nvarcharCol2"));
assertEquals(longNvarcharValue2, rs.getString("longnvarcharCol2"));
assertFalse(rs.next());
}
}
Expand All @@ -1493,12 +1536,15 @@ private void getCreateTableWithStringData() throws SQLException {
try (Statement stmt = connection.createStatement()) {
TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(tableNameBulkString), stmt);
String createTableSQL = "CREATE TABLE " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString) + " (" +
"charCol CHAR(8) NOT NULL, " +
"varcharCol VARCHAR(50) NOT NULL, " +
"longvarcharCol VARCHAR(MAX) NOT NULL, " +
"ncharCol NCHAR(9) NOT NULL, " +
"nvarcharCol NVARCHAR(50) NOT NULL, " +
"longnvarcharCol NVARCHAR(MAX) NOT NULL" + ")";
"charCol CHAR(8), " +
"varcharCol VARCHAR(50), " +
"longvarcharCol VARCHAR(MAX), " +
"ncharCol1 NCHAR(9), " +
"nvarcharCol1 NVARCHAR(50), " +
"longnvarcharCol1 NVARCHAR(MAX), " +
"ncharCol2 NCHAR(9), " +
"nvarcharCol2 NVARCHAR(50), " +
"longnvarcharCol2 NVARCHAR(MAX)" + ")";

stmt.execute(createTableSQL);
}
Expand Down
Loading