Properly Insert Accented Characters When Using Bulk Copy and Non-Unicode Strings (#2727)

muskan124947 · web-flow · commit 1dc9eb773935 · 2025-08-08T09:57:11.000+05:30
* Fix to handle accented characters in database when using useBulkCopyForBatchInsert=true and sendStringParametersAsUnicode=false

* Addressed comments

* Updated test case to verify results with all combinations for useBulkCopyForBatchInsert and sendStringParametersAsUnicode property
diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkBatchInsertRecord.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkBatchInsertRecord.java
@@ -7,6 +7,7 @@
 
 import java.math.BigDecimal;
 import java.math.RoundingMode;
+import java.nio.charset.Charset;
 import java.sql.Types;
 import java.text.DecimalFormat;
 import java.text.MessageFormat;
@@ -34,6 +35,7 @@ class SQLServerBulkBatchInsertRecord extends SQLServerBulkRecord {
     private int batchParamIndex = -1;
     private List<String> columnList;
     private List<String> valueList;
+    private Charset charset;
 
     /*
      * Class name for logging.
@@ -44,10 +46,10 @@ class SQLServerBulkBatchInsertRecord extends SQLServerBulkRecord {
      * Constructs a SQLServerBulkBatchInsertRecord with the batch parameter, column list, value list, and encoding
      */
     SQLServerBulkBatchInsertRecord(ArrayList<Parameter[]> batchParam, ArrayList<String> columnList,
-            ArrayList<String> valueList, String encoding, boolean columnNameCaseSensitive) throws SQLServerException {
+            ArrayList<String> valueList, Charset charset, boolean columnNameCaseSensitive) throws SQLServerException {
         initLoggerResources();
         if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) {
-            loggerExternal.entering(loggerPackageName, loggerClassName, new Object[] {batchParam, encoding});
+            loggerExternal.entering(loggerPackageName, loggerClassName, new Object[] {batchParam, charset.name()});
         }
 
         if (null == batchParam) {
@@ -62,6 +64,7 @@ class SQLServerBulkBatchInsertRecord extends SQLServerBulkRecord {
         this.columnList = columnList;
         this.valueList = valueList;
         this.columnNameCaseSensitive = columnNameCaseSensitive;
+        this.charset = charset;
         columnMetadata = new HashMap<>();
 
         loggerExternal.exiting(loggerPackageName, loggerClassName);
@@ -193,7 +196,7 @@ else if (dateTimeFormatter != null)
                  * If the data is already a string, return it as is.
                  */
                 if (data instanceof byte[]) {
-                    return new String((byte[]) data);
+                    return new String((byte[]) data, charset);
                 }
                 return data;
             }
diff --git a/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java b/src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java
@@ -2218,7 +2218,8 @@ public int[] executeBatch() throws SQLServerException, BatchUpdateException, SQL
                         }
 
                         SQLServerBulkBatchInsertRecord batchRecord = new SQLServerBulkBatchInsertRecord(
-                                batchParamValues, bcOperationColumnList, bcOperationValueList, null, isDBColationCaseSensitive());
+                                batchParamValues, bcOperationColumnList, bcOperationValueList, 
+                                connection.getDatabaseCollation().getCharset(), isDBColationCaseSensitive());
 
                         for (int i = 1; i <= rs.getColumnCount(); i++) {
                             Column c = rs.getColumn(i);
@@ -2426,7 +2427,8 @@ public long[] executeLargeBatch() throws SQLServerException, BatchUpdateExceptio
                         }
 
                         SQLServerBulkBatchInsertRecord batchRecord = new SQLServerBulkBatchInsertRecord(
-                                batchParamValues, bcOperationColumnList, bcOperationValueList, null, isDBColationCaseSensitive());
+                                batchParamValues, bcOperationColumnList, bcOperationValueList, 
+                                connection.getDatabaseCollation().getCharset(), isDBColationCaseSensitive());
 
                         for (int i = 1; i <= rs.getColumnCount(); i++) {
                             Column c = rs.getColumn(i);
diff --git a/src/test/java/com/microsoft/sqlserver/jdbc/preparedStatement/BatchExecutionWithBulkCopyTest.java b/src/test/java/com/microsoft/sqlserver/jdbc/preparedStatement/BatchExecutionWithBulkCopyTest.java
@@ -1408,82 +1408,76 @@ private void getCreateTableTemporalSQL(String tableName) throws SQLException {
     }
 
     /**
-     * Test batch insert using bulk copy with string values when setSendStringParametersAsUnicode is true.
+     * Test string values using prepared statement using accented and unicode characters.
+     * This test covers all combinations of useBulkCopyForBatchInsert and sendStringParametersAsUnicode.
+     * 
+     * @throws Exception
      */
     @Test
-    public void testBulkInsertStringWhenSentAsUnicode() throws Exception {
-        String insertSQL = "INSERT INTO " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString)
-                + " (charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol) VALUES (?, ?, ?, ?, ?, ?)";
-
-        String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol FROM "
-                + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString);
-
-        try (Connection connection = PrepUtil.getConnection(
-                connectionString + ";useBulkCopyForBatchInsert=true;sendStringParametersAsUnicode=true;");
-                SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
-                Statement stmt = (SQLServerStatement) connection.createStatement()) {
-
-            getCreateTableWithStringData();
-
-            pstmt.setString(1, "CHAR_VAL");
-            pstmt.setString(2, "VARCHAR_VALUE");
-            pstmt.setString(3, "LONGVARCHAR_VALUE_WITH_MORE_TEXT");
-            pstmt.setString(4, "NCHAR_VAL");
-            pstmt.setString(5, "NVARCHAR_VALUE");
-            pstmt.setString(6, "LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT");
-            pstmt.addBatch();
-            pstmt.executeBatch();
-
-            // Validate inserted data
-            try (ResultSet rs = stmt.executeQuery(selectSQL)) {
-                assertTrue(rs.next(), "Expected at least one row in result set");
-                assertEquals("CHAR_VAL", rs.getString("charCol"));
-                assertEquals("VARCHAR_VALUE", rs.getString("varcharCol"));
-                assertEquals("LONGVARCHAR_VALUE_WITH_MORE_TEXT", rs.getString("longvarcharCol"));
-                assertEquals("NCHAR_VAL", rs.getString("ncharCol"));
-                assertEquals("NVARCHAR_VALUE", rs.getString("nvarcharCol"));
-                assertEquals("LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT", rs.getString("longnvarcharCol"));
-                assertFalse(rs.next());
+    public void testBulkInsertStringAllCombinations() throws Exception {
+        boolean[] bulkCopyOptions = { true, false };
+        boolean[] unicodeOptions = { true, false };
+        for (boolean useBulkCopy : bulkCopyOptions) {
+            for (boolean sendUnicode : unicodeOptions) {
+                runBulkInsertStringTest(useBulkCopy, sendUnicode);
             }
         }
     }
 
     /**
-     * Test batch insert using bulk copy with string values when setSendStringParametersAsUnicode is false.
+     * Test batch insert using accented and unicode characters.
      */
-    @Test
-    public void testBulkInsertStringWhenNotSentAsUnicode() throws Exception {
+    public void runBulkInsertStringTest(boolean useBulkCopy, boolean sendUnicode) throws Exception {
         String insertSQL = "INSERT INTO " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString)
-                + " (charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol) VALUES (?, ?, ?, ?, ?, ?)";
+                + " (charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, longnvarcharCol1, "
+                + "ncharCol2, nvarcharCol2, longnvarcharCol2) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)";
 
-        String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol, nvarcharCol, longnvarcharCol FROM "
+        String selectSQL = "SELECT charCol, varcharCol, longvarcharCol, ncharCol1, nvarcharCol1, "
+                + "longnvarcharCol1, ncharCol2, nvarcharCol2, longnvarcharCol2 FROM "
                 + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString);
 
         try (Connection connection = PrepUtil.getConnection(
-                connectionString + ";useBulkCopyForBatchInsert=true;sendStringParametersAsUnicode=false;");
+                connectionString + ";useBulkCopyForBatchInsert=" + useBulkCopy + ";sendStringParametersAsUnicode="
+                        + sendUnicode + ";");
                 SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) connection.prepareStatement(insertSQL);
                 Statement stmt = (SQLServerStatement) connection.createStatement()) {
 
             getCreateTableWithStringData();
 
-            pstmt.setString(1, "CHAR_VAL");
-            pstmt.setString(2, "VARCHAR_VALUE");
-            pstmt.setString(3, "LONGVARCHAR_VALUE_WITH_MORE_TEXT");
-            pstmt.setString(4, "NCHAR_VAL");
-            pstmt.setString(5, "NVARCHAR_VALUE");
-            pstmt.setString(6, "LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT");
+            String charValue = "Anaïs_Ni";
+            String varcharValue = "café";
+            String longVarcharValue = "Sørén Kierkégaard";
+            String ncharValue1 = "José Müll";
+            String nvarcharValue1 = "José Müller";
+            String longNvarcharValue1 = "François Saldaña";
+            String ncharValue2 = "Test1汉字😀";
+            String nvarcharValue2 = "汉字";
+            String longNvarcharValue2 = "日本語";
+
+            pstmt.setString(1, charValue);
+            pstmt.setString(2, varcharValue);
+            pstmt.setString(3, longVarcharValue);
+            pstmt.setString(4, ncharValue1);
+            pstmt.setString(5, nvarcharValue1);
+            pstmt.setString(6, longNvarcharValue1);
+            pstmt.setNString(7, ncharValue2);
+            pstmt.setNString(8, nvarcharValue2);
+            pstmt.setNString(9, longNvarcharValue2);
             pstmt.addBatch();
             pstmt.executeBatch();
 
             // Validate inserted data
             try (ResultSet rs = stmt.executeQuery(selectSQL)) {
                 assertTrue(rs.next(), "Expected at least one row in result set");
-                assertEquals("CHAR_VAL", rs.getString("charCol"));
-                assertEquals("VARCHAR_VALUE", rs.getString("varcharCol"));
-                assertEquals("LONGVARCHAR_VALUE_WITH_MORE_TEXT", rs.getString("longvarcharCol"));
-                assertEquals("NCHAR_VAL", rs.getString("ncharCol"));
-                assertEquals("NVARCHAR_VALUE", rs.getString("nvarcharCol"));
-                assertEquals("LONGNVARCHAR_VALUE_WITH_UNICODE_TEXT", rs.getString("longnvarcharCol"));
+                assertEquals(charValue, rs.getString("charCol"));
+                assertEquals(varcharValue, rs.getString("varcharCol"));
+                assertEquals(longVarcharValue, rs.getString("longvarcharCol"));
+                assertEquals(ncharValue1, rs.getString("ncharCol1"));
+                assertEquals(nvarcharValue1, rs.getString("nvarcharCol1"));
+                assertEquals(longNvarcharValue1, rs.getString("longnvarcharCol1"));
+                assertEquals(ncharValue2, rs.getString("ncharCol2"));
+                assertEquals(nvarcharValue2, rs.getString("nvarcharCol2"));
+                assertEquals(longNvarcharValue2, rs.getString("longnvarcharCol2"));
                 assertFalse(rs.next());
             }
         }
@@ -1493,12 +1487,15 @@ private void getCreateTableWithStringData() throws SQLException {
         try (Statement stmt = connection.createStatement()) {
             TestUtils.dropTableIfExists(AbstractSQLGenerator.escapeIdentifier(tableNameBulkString), stmt);
             String createTableSQL = "CREATE TABLE " + AbstractSQLGenerator.escapeIdentifier(tableNameBulkString) + " (" +
-                    "charCol CHAR(8) NOT NULL, " +
-                    "varcharCol VARCHAR(50) NOT NULL, " +
-                    "longvarcharCol VARCHAR(MAX) NOT NULL, " +
-                    "ncharCol NCHAR(9) NOT NULL, " +
-                    "nvarcharCol NVARCHAR(50) NOT NULL, " +
-                    "longnvarcharCol NVARCHAR(MAX) NOT NULL" + ")";
+                    "charCol CHAR(8), " +
+                    "varcharCol VARCHAR(50), " +
+                    "longvarcharCol VARCHAR(MAX), " +
+                    "ncharCol1 NCHAR(9), " +
+                    "nvarcharCol1 NVARCHAR(50), " +
+                    "longnvarcharCol1 NVARCHAR(MAX), " +
+                    "ncharCol2 NCHAR(9), " +
+                    "nvarcharCol2 NVARCHAR(50), " +
+                    "longnvarcharCol2 NVARCHAR(MAX)" + ")";
 
             stmt.execute(createTableSQL);
         }
