Skip to content

Commit 8a8a953

Browse files
authored
[core] Correct bitmap types support in writer (#5309)
1 parent 0b5d8da commit 8a8a953

File tree

8 files changed

+348
-315
lines changed

8 files changed

+348
-315
lines changed

docs/content/concepts/spec/fileindex.md

Lines changed: 0 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -261,34 +261,6 @@ Bitmap only support the following data type:
261261
<td><code>BooleanType</code></td>
262262
<td>true</td>
263263
</tr>
264-
<tr>
265-
<td><code>DecimalType(precision, scale)</code></td>
266-
<td>false</td>
267-
</tr>
268-
<tr>
269-
<td><code>FloatType</code></td>
270-
<td>Not recommended</td>
271-
</tr>
272-
<tr>
273-
<td><code>DoubleType</code></td>
274-
<td>Not recommended</td>
275-
</tr>
276-
<tr>
277-
<td><code>VarBinaryType</code>, <code>BinaryType</code></td>
278-
<td>false</td>
279-
</tr>
280-
<tr>
281-
<td><code>RowType</code></td>
282-
<td>false</td>
283-
</tr>
284-
<tr>
285-
<td><code>MapType</code></td>
286-
<td>false</td>
287-
</tr>
288-
<tr>
289-
<td><code>ArrayType</code></td>
290-
<td>false</td>
291-
</tr>
292264
</tbody>
293265
</table>
294266

@@ -384,37 +356,5 @@ BSI only support the following data type:
384356
<td><code>DecimalType(precision, scale)</code></td>
385357
<td>true</td>
386358
</tr>
387-
<tr>
388-
<td><code>FloatType</code></td>
389-
<td>false</td>
390-
</tr>
391-
<tr>
392-
<td><code>DoubleType</code></td>
393-
<td>false</td>
394-
</tr>
395-
<tr>
396-
<td><code>String</code></td>
397-
<td>false</td>
398-
</tr>
399-
<tr>
400-
<td><code>VarBinaryType</code>, <code>BinaryType</code></td>
401-
<td>false</td>
402-
</tr>
403-
<tr>
404-
<td><code>RowType</code></td>
405-
<td>false</td>
406-
</tr>
407-
<tr>
408-
<td><code>MapType</code></td>
409-
<td>false</td>
410-
</tr>
411-
<tr>
412-
<td><code>ArrayType</code></td>
413-
<td>false</td>
414-
</tr>
415-
<tr>
416-
<td><code>BooleanType</code></td>
417-
<td>false</td>
418-
</tr>
419359
</tbody>
420360
</table>

paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexWriter.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ public void writeRecord(Object key) {
2828
write(key);
2929
}
3030

31+
/** The key object may be reused, if saved in memory, please be sure to manually copy it. */
3132
public abstract void write(Object key);
3233

3334
public abstract byte[] serializedBytes();

paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapFileIndex.java

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
package org.apache.paimon.fileindex.bitmap;
2020

21+
import org.apache.paimon.data.BinaryString;
2122
import org.apache.paimon.data.Timestamp;
2223
import org.apache.paimon.fileindex.FileIndexReader;
2324
import org.apache.paimon.fileindex.FileIndexResult;
@@ -27,7 +28,6 @@
2728
import org.apache.paimon.options.Options;
2829
import org.apache.paimon.predicate.FieldRef;
2930
import org.apache.paimon.types.DataType;
30-
import org.apache.paimon.types.DataTypeDefaultVisitor;
3131
import org.apache.paimon.types.LocalZonedTimestampType;
3232
import org.apache.paimon.types.TimestampType;
3333
import org.apache.paimon.utils.RoaringBitmap32;
@@ -315,7 +315,53 @@ private void readInternalMeta(DataType dataType) {
315315
// Currently, it is mainly used to convert timestamps to long
316316
public static Function<Object, Object> getValueMapper(DataType dataType) {
317317
return dataType.accept(
318-
new DataTypeDefaultVisitor<Function<Object, Object>>() {
318+
new BitmapTypeVisitor<Function<Object, Object>>() {
319+
320+
@Override
321+
public Function<Object, Object> visitBinaryString() {
322+
return o -> {
323+
if (o instanceof BinaryString) {
324+
return ((BinaryString) o).copy();
325+
}
326+
return o;
327+
};
328+
}
329+
330+
@Override
331+
public Function<Object, Object> visitByte() {
332+
return Function.identity();
333+
}
334+
335+
@Override
336+
public Function<Object, Object> visitShort() {
337+
return Function.identity();
338+
}
339+
340+
@Override
341+
public Function<Object, Object> visitInt() {
342+
return Function.identity();
343+
}
344+
345+
@Override
346+
public Function<Object, Object> visitLong() {
347+
return Function.identity();
348+
}
349+
350+
@Override
351+
public Function<Object, Object> visitFloat() {
352+
return Function.identity();
353+
}
354+
355+
@Override
356+
public Function<Object, Object> visitDouble() {
357+
return Function.identity();
358+
}
359+
360+
@Override
361+
public Function<Object, Object> visitBoolean() {
362+
return Function.identity();
363+
}
364+
319365
@Override
320366
public Function<Object, Object> visit(TimestampType timestampType) {
321367
return getTimeStampMapper(timestampType.getPrecision());
@@ -327,11 +373,6 @@ public Function<Object, Object> visit(
327373
return getTimeStampMapper(localZonedTimestampType.getPrecision());
328374
}
329375

330-
@Override
331-
protected Function<Object, Object> defaultMethod(DataType dataType) {
332-
return Function.identity();
333-
}
334-
335376
private Function<Object, Object> getTimeStampMapper(int precision) {
336377
return o -> {
337378
if (o == null) {

0 commit comments

Comments
 (0)