Skip to content

Commit a762d7c

Browse files
hannes92hannes92
authored
Performance optimizations when bulk loading large amounts of timestamps (#2194)
1 parent 974e4a1 commit a762d7c

File tree

3 files changed

+108
-22
lines changed

3 files changed

+108
-22
lines changed

‎src/main/java/com/microsoft/sqlserver/jdbc/IOBuffer.java

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import java.sql.Timestamp;
4141
import java.text.MessageFormat;
4242
import java.time.LocalDate;
43+
import java.time.LocalDateTime;
4344
import java.time.OffsetDateTime;
4445
import java.time.OffsetTime;
4546
import java.util.ArrayList;
@@ -3755,29 +3756,24 @@ void writeSmalldatetime(String value) throws SQLServerException {
37553756
writeShort((short) minutesSinceMidnight);
37563757
}
37573758

3758-
void writeDatetime(String value) throws SQLServerException {
3759-
GregorianCalendar calendar = initializeCalender(TimeZone.getDefault());
3760-
long utcMillis; // Value to which the calendar is to be set (in milliseconds 1/1/1970 00:00:00 GMT)
3759+
void writeDatetime(java.sql.Timestamp dateValue) throws SQLServerException {
3760+
LocalDateTime ldt = dateValue.toLocalDateTime();
37613761
int subSecondNanos;
3762-
java.sql.Timestamp timestampValue = java.sql.Timestamp.valueOf(value);
3763-
utcMillis = timestampValue.getTime();
3764-
subSecondNanos = timestampValue.getNanos();
3762+
subSecondNanos = ldt.getNano();
37653763

3766-
// Load the calendar with the desired value
3767-
calendar.setTimeInMillis(utcMillis);
37683764

37693765
// Number of days there have been since the SQL Base Date.
37703766
// These are based on SQL Server algorithms
3771-
int daysSinceSQLBaseDate = DDC.daysSinceBaseDate(calendar.get(Calendar.YEAR),
3772-
calendar.get(Calendar.DAY_OF_YEAR), TDS.BASE_YEAR_1900);
3767+
int daysSinceSQLBaseDate = DDC.daysSinceBaseDate(ldt.getYear(),
3768+
ldt.getDayOfYear(), TDS.BASE_YEAR_1900);
37733769

37743770
// Number of milliseconds since midnight of the current day.
37753771
int millisSinceMidnight = (subSecondNanos + Nanos.PER_MILLISECOND / 2) / Nanos.PER_MILLISECOND + // Millis into
3776-
// the current
3777-
// second
3778-
1000 * calendar.get(Calendar.SECOND) + // Seconds into the current minute
3779-
60 * 1000 * calendar.get(Calendar.MINUTE) + // Minutes into the current hour
3780-
60 * 60 * 1000 * calendar.get(Calendar.HOUR_OF_DAY); // Hours into the current day
3772+
// the current
3773+
// second
3774+
1000 * ldt.getSecond() + // Seconds into the current minute
3775+
60 * 1000 * ldt.getMinute() + // Minutes into the current hour
3776+
60 * 60 * 1000 * ldt.getHour(); // Hours into the current day
37813777

37823778
// The last millisecond of the current day is always rounded to the first millisecond
37833779
// of the next day because DATETIME is only accurate to 1/300th of a second.

‎src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkCopy.java

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2463,7 +2463,12 @@ else if (null != sourceCryptoMeta) {
24632463
case DATETIME:
24642464
if (bulkNullable)
24652465
tdsWriter.writeByte((byte) 0x08);
2466-
tdsWriter.writeDatetime(colValue.toString());
2466+
2467+
if (colValue instanceof java.sql.Timestamp) {
2468+
tdsWriter.writeDatetime((java.sql.Timestamp) colValue);
2469+
} else {
2470+
tdsWriter.writeDatetime(java.sql.Timestamp.valueOf(colValue.toString()));
2471+
}
24672472
break;
24682473
default: // DATETIME2
24692474
if (2 >= bulkScale)
@@ -2697,15 +2702,15 @@ private void writeSqlVariant(TDSWriter tdsWriter, Object colValue, ResultSet sou
26972702
tdsWriter.writeTime((java.sql.Timestamp) colValue, timeBulkScale);
26982703
break;
26992704

2700-
case DATETIME8:
2701-
writeBulkCopySqlVariantHeader(10, TDSType.DATETIME8.byteValue(), (byte) 0, tdsWriter);
2702-
tdsWriter.writeDatetime(colValue.toString());
2703-
break;
2704-
27052705
case DATETIME4:
27062706
// when the type is ambiguous, we write to bigger type
2707+
case DATETIME8:
27072708
writeBulkCopySqlVariantHeader(10, TDSType.DATETIME8.byteValue(), (byte) 0, tdsWriter);
2708-
tdsWriter.writeDatetime(colValue.toString());
2709+
if (colValue instanceof java.sql.Timestamp) {
2710+
tdsWriter.writeDatetime((java.sql.Timestamp) colValue);
2711+
} else {
2712+
tdsWriter.writeDatetime(java.sql.Timestamp.valueOf(colValue.toString()));
2713+
}
27092714
break;
27102715

27112716
case DATETIME2N:

‎src/test/java/com/microsoft/sqlserver/jdbc/bulkCopy/BulkCopyAllTypesTest.java

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
import java.sql.ResultSet;
99
import java.sql.SQLException;
1010
import java.sql.Statement;
11+
import java.sql.Timestamp;
12+
import java.sql.Types;
1113

1214
import org.junit.jupiter.api.BeforeAll;
1315
import org.junit.jupiter.api.Tag;
@@ -16,15 +18,33 @@
1618
import org.junit.runner.RunWith;
1719

1820
import com.microsoft.sqlserver.jdbc.ComparisonUtil;
21+
import com.microsoft.sqlserver.jdbc.RandomData;
22+
import com.microsoft.sqlserver.jdbc.RandomUtil;
1923
import com.microsoft.sqlserver.jdbc.SQLServerBulkCopy;
24+
import com.microsoft.sqlserver.jdbc.SQLServerBulkCopyOptions;
2025
import com.microsoft.sqlserver.jdbc.TestUtils;
26+
import com.microsoft.sqlserver.testframework.AbstractSQLGenerator;
2127
import com.microsoft.sqlserver.testframework.AbstractTest;
2228
import com.microsoft.sqlserver.testframework.Constants;
2329
import com.microsoft.sqlserver.testframework.DBConnection;
2430
import com.microsoft.sqlserver.testframework.DBStatement;
2531
import com.microsoft.sqlserver.testframework.DBTable;
2632
import com.microsoft.sqlserver.testframework.PrepUtil;
2733

34+
import javax.sql.RowSetMetaData;
35+
import javax.sql.rowset.CachedRowSet;
36+
import javax.sql.rowset.RowSetFactory;
37+
import javax.sql.rowset.RowSetMetaDataImpl;
38+
import javax.sql.rowset.RowSetProvider;
39+
40+
import java.util.ArrayList;
41+
import java.util.List;
42+
import java.util.stream.Collectors;
43+
import java.util.stream.IntStream;
44+
45+
import static org.junit.Assert.assertEquals;
46+
import static org.junit.Assert.assertTrue;
47+
2848

2949
@RunWith(JUnitPlatform.class)
3050
public class BulkCopyAllTypesTest extends AbstractTest {
@@ -98,6 +118,71 @@ private void terminateVariation() throws SQLException {
98118
try (Statement stmt = connection.createStatement()) {
99119
TestUtils.dropTableIfExists(tableSrc.getEscapedTableName(), stmt);
100120
TestUtils.dropTableIfExists(tableDest.getEscapedTableName(), stmt);
121+
TestUtils.dropTableIfExists(dateTimeTestTable, stmt);
122+
}
123+
}
124+
125+
private static final int DATETIME_COL_COUNT = 2;
126+
private static final int DATETIME_ROW_COUNT = 1;
127+
private static final String dateTimeTestTable =
128+
AbstractSQLGenerator.escapeIdentifier(RandomUtil.getIdentifier("bulkCopyTimestampTest"));
129+
130+
@Test
131+
public void testBulkCopyTimestamp() throws SQLException {
132+
List<Timestamp> timeStamps = new ArrayList<>();
133+
try (Connection con = getConnection(); Statement stmt = connection.createStatement()) {
134+
String colSpec = IntStream.range(1, DATETIME_COL_COUNT + 1).mapToObj(x -> String.format("c%d datetime", x)).collect(
135+
Collectors.joining(","));
136+
String sql1 = String.format("create table %s (%s)", dateTimeTestTable, colSpec);
137+
stmt.execute(sql1);
138+
139+
RowSetFactory rsf = RowSetProvider.newFactory();
140+
CachedRowSet crs = rsf.createCachedRowSet();
141+
RowSetMetaData rsmd = new RowSetMetaDataImpl();
142+
rsmd.setColumnCount(DATETIME_COL_COUNT);
143+
144+
for (int i = 1; i <= DATETIME_COL_COUNT; i++) {
145+
rsmd.setColumnName(i, String.format("c%d", i));
146+
rsmd.setColumnType(i, Types.TIMESTAMP);
147+
}
148+
crs.setMetaData(rsmd);
149+
150+
for (int i = 0; i < DATETIME_COL_COUNT; i++) {
151+
timeStamps.add(RandomData.generateDatetime(false));
152+
}
153+
154+
for (int ri = 0; ri < DATETIME_ROW_COUNT; ri++) {
155+
crs.moveToInsertRow();
156+
157+
for (int i = 1; i <= DATETIME_COL_COUNT; i++) {
158+
crs.updateTimestamp(i, timeStamps.get(i - 1));
159+
}
160+
crs.insertRow();
161+
}
162+
crs.moveToCurrentRow();
163+
164+
try (SQLServerBulkCopy bcOperation = new SQLServerBulkCopy(con)) {
165+
SQLServerBulkCopyOptions bcOptions = new SQLServerBulkCopyOptions();
166+
bcOptions.setBatchSize(5000);
167+
bcOperation.setDestinationTableName(dateTimeTestTable);
168+
bcOperation.setBulkCopyOptions(bcOptions);
169+
bcOperation.writeToServer(crs);
170+
}
171+
172+
try (ResultSet rs = stmt.executeQuery("select * from " + dateTimeTestTable)) {
173+
assertTrue(rs.next());
174+
175+
for (int i = 1; i <= DATETIME_COL_COUNT; i++) {
176+
long expectedTimestamp = getTime(timeStamps.get(i - 1));
177+
long actualTimestamp = getTime(rs.getTimestamp(i));
178+
179+
assertEquals(expectedTimestamp, actualTimestamp);
180+
}
181+
}
101182
}
102183
}
184+
185+
private static long getTime(Timestamp time) {
186+
return (3 * time.getTime() + 5) / 10;
187+
}
103188
}

0 commit comments

Comments
 (0)