Skip to content

Commit c8b28a1

Browse files
authored
fix: Always use setSafe when writing values from spark (#4367)
There's likely a way to preallocate the memory once but it's not quite obvious how Signed-off-by: Robert Kruszewski <[email protected]>
1 parent 2d909dc commit c8b28a1

File tree

1 file changed

+8
-8
lines changed

1 file changed

+8
-8
lines changed

java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -191,19 +191,19 @@ private void writeBatch() throws IOException {
191191
private void populateVector(
192192
FieldVector vector, DataType dataType, SpecializedGetters row, int fieldIndex, int rowIndex) {
193193
if (dataType instanceof BooleanType) {
194-
((BitVector) vector).set(rowIndex, row.getBoolean(fieldIndex) ? 1 : 0);
194+
((BitVector) vector).setSafe(rowIndex, row.getBoolean(fieldIndex) ? 1 : 0);
195195
} else if (dataType instanceof ByteType) {
196-
((TinyIntVector) vector).set(rowIndex, row.getByte(fieldIndex));
196+
((TinyIntVector) vector).setSafe(rowIndex, row.getByte(fieldIndex));
197197
} else if (dataType instanceof ShortType) {
198-
((SmallIntVector) vector).set(rowIndex, row.getShort(fieldIndex));
198+
((SmallIntVector) vector).setSafe(rowIndex, row.getShort(fieldIndex));
199199
} else if (dataType instanceof IntegerType) {
200-
((IntVector) vector).set(rowIndex, row.getInt(fieldIndex));
200+
((IntVector) vector).setSafe(rowIndex, row.getInt(fieldIndex));
201201
} else if (dataType instanceof LongType) {
202-
((BigIntVector) vector).set(rowIndex, row.getLong(fieldIndex));
202+
((BigIntVector) vector).setSafe(rowIndex, row.getLong(fieldIndex));
203203
} else if (dataType instanceof FloatType) {
204-
((Float4Vector) vector).set(rowIndex, row.getFloat(fieldIndex));
204+
((Float4Vector) vector).setSafe(rowIndex, row.getFloat(fieldIndex));
205205
} else if (dataType instanceof DoubleType) {
206-
((Float8Vector) vector).set(rowIndex, row.getDouble(fieldIndex));
206+
((Float8Vector) vector).setSafe(rowIndex, row.getDouble(fieldIndex));
207207
} else if (dataType instanceof StringType) {
208208
UTF8String str = row.getUTF8String(fieldIndex);
209209
if (str != null) {
@@ -220,7 +220,7 @@ private void populateVector(
220220
// Use Decimal type from InternalRow
221221
java.math.BigDecimal decimal = row.getDecimal(fieldIndex, decType.precision(), decType.scale())
222222
.toJavaBigDecimal();
223-
((DecimalVector) vector).set(rowIndex, decimal);
223+
((DecimalVector) vector).setSafe(rowIndex, decimal);
224224
}
225225
} else if (dataType instanceof ArrayType) {
226226
ArrayType arrayType = (ArrayType) dataType;

0 commit comments

Comments
 (0)