Skip to content

Commit 54543af

Browse files
manticore-projectsmanticore-projects
committed
fix: issues with large Decimals and Oracle variable scale
- fix large negative decimals - add test for large negative decimals
1 parent 8fbbe50 commit 54543af

File tree

2 files changed

+88
-20
lines changed

2 files changed

+88
-20
lines changed

‎src/main/java/com/manticore/jdbc/parquetwriter/JDBCParquetWriter.java‎

Lines changed: 38 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import java.sql.Statement;
4242
import java.sql.Time;
4343
import java.sql.Timestamp;
44+
import java.util.Arrays;
4445
import java.util.logging.Logger;
4546

4647
/**
@@ -293,18 +294,26 @@ public static long write(File f, String tableName, ResultSet rs,
293294
if (scale > 0 && precision <= 18) {
294295
group.add(columnName, decimal.unscaledValue().longValue());
295296
} else if (scale > 0) {
296-
byte[] bytes =
297-
decimal.setScale(scale, RoundingMode.HALF_EVEN)
298-
.unscaledValue().toByteArray();
297+
// Scale the decimal to the desired precision and scale
298+
byte[] unscaledBytes = decimal
299+
.setScale(scale, RoundingMode.HALF_EVEN)
300+
.unscaledValue()
301+
.toByteArray();
302+
int requiredBytes = 16;
299303

300-
// Ensure the byte array is padded correctly for the precision
301-
int numBytes =
302-
Math.max((int) Math.ceil(precision / Math.log10(2) / 8),
303-
bytes.length); // Calculate required bytes
304-
byte[] paddedBytes = new byte[numBytes];
305-
System.arraycopy(bytes, 0, paddedBytes, numBytes - bytes.length,
306-
bytes.length);
304+
// Ensure the byte array is padded correctly with sign extension
305+
byte[] paddedBytes = new byte[requiredBytes];
306+
byte signByte = (unscaledBytes[0] < 0) ? (byte) 0xFF : 0x00; // Extend
307+
// sign
308+
// bit
309+
Arrays.fill(paddedBytes, 0,
310+
requiredBytes - unscaledBytes.length, signByte);
311+
System.arraycopy(unscaledBytes, 0, paddedBytes,
312+
requiredBytes - unscaledBytes.length,
313+
unscaledBytes.length);
307314

315+
// Add the binary representation of the decimal to the Parquet
316+
// group
308317
group.add(columnName,
309318
Binary.fromConstantByteArray(paddedBytes));
310319
} else if (scale == -127) {
@@ -317,17 +326,27 @@ public static long write(File f, String tableName, ResultSet rs,
317326
*/
318327
precision = 38;
319328
scale = 10;
320-
byte[] bytes = decimal.setScale(scale, RoundingMode.HALF_EVEN)
321-
.unscaledValue().toByteArray();
322329

323-
// Ensure the byte array is padded correctly for the precision
324-
int numBytes =
325-
Math.max((int) Math.ceil(precision / Math.log10(2) / 8),
326-
bytes.length); // Calculate required bytes
327-
byte[] paddedBytes = new byte[numBytes];
328-
System.arraycopy(bytes, 0, paddedBytes, numBytes - bytes.length,
329-
bytes.length);
330+
// Scale the decimal to the desired precision and scale
331+
byte[] unscaledBytes = decimal
332+
.setScale(scale, RoundingMode.HALF_EVEN)
333+
.unscaledValue()
334+
.toByteArray();
335+
int requiredBytes = 16;
330336

337+
// Ensure the byte array is padded correctly with sign extension
338+
byte[] paddedBytes = new byte[requiredBytes];
339+
byte signByte = (unscaledBytes[0] < 0) ? (byte) 0xFF : 0x00; // Extend
340+
// sign
341+
// bit
342+
Arrays.fill(paddedBytes, 0,
343+
requiredBytes - unscaledBytes.length, signByte);
344+
System.arraycopy(unscaledBytes, 0, paddedBytes,
345+
requiredBytes - unscaledBytes.length,
346+
unscaledBytes.length);
347+
348+
// Add the binary representation of the decimal to the Parquet
349+
// group
331350
group.add(columnName,
332351
Binary.fromConstantByteArray(paddedBytes));
333352
} else if (precision < 5) {

‎src/test/java/com/manticore/jdbc/parquetwriter/JDBCParquetWriterTest.java‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.junit.jupiter.api.Test;
2323

2424
import java.io.File;
25+
import java.math.BigDecimal;
2526
import java.sql.Connection;
2627
import java.sql.DriverManager;
2728
import java.sql.ResultSet;
@@ -90,7 +91,6 @@ void write() throws Exception {
9091
}
9192

9293
@Test
93-
@Disabled
9494
void testVBoxQuery() throws Exception {
9595
Properties properties = new Properties();
9696
properties.put("user", "SA");
@@ -117,6 +117,55 @@ void testVBoxQuery() throws Exception {
117117

118118
}
119119

120+
@Test
121+
void testBigDecimal() throws Exception {
122+
String tableName = "test";
123+
String decimalStr = "-24999999999999.99500";
124+
125+
File file = File.createTempFile(tableName, ".parquet");
126+
try (Connection conn = DriverManager.getConnection(
127+
"jdbc:h2:mem:test")) {
128+
129+
String[] ddlStr = {
130+
"CREATE TABLE decimal_test (\n"
131+
+ " amount DECIMAL(23,5) NULL\n"
132+
+ ")\n"
133+
+ ";"
134+
135+
, "INSERT INTO decimal_test \n"
136+
+ "VALUES (" + decimalStr + ");"
137+
};
138+
139+
try (Statement st = conn.createStatement()) {
140+
for (String sqlStr : ddlStr) {
141+
st.execute(sqlStr);
142+
}
143+
}
144+
145+
long writtenRows = 0;
146+
String sqlStr = "SELECT *\n"
147+
+ "FROM decimal_test";
148+
try (Statement st = conn.createStatement(); ResultSet rs = st.executeQuery(sqlStr);) {
149+
writtenRows = JDBCParquetWriter.write(file, tableName, rs);
150+
}
151+
}
152+
153+
String sqlStr = "SELECT *\n"
154+
+ "FROM '" + file.getAbsolutePath() + "';";
155+
try (Connection conn = DriverManager.getConnection(
156+
"jdbc:duckdb:");
157+
Statement st = conn.createStatement();
158+
ResultSet rs = st.executeQuery(sqlStr)) {
159+
if (rs.next()) {
160+
BigDecimal actualDecimal = rs.getBigDecimal(1);
161+
Assertions.assertEquals(decimalStr, actualDecimal.toPlainString());
162+
163+
final boolean delete = file.delete();
164+
}
165+
}
166+
167+
}
168+
120169
@Test
121170
@Disabled
122171
void getParquetSchemaFromResultSet() {}

0 commit comments

Comments
 (0)