Skip to content

Commit b0d0b00

Browse files
committed
[Bug](catalog) fix runtime filter partition pruning error with binary type
1 parent c48244a commit b0d0b00

File tree

5 files changed

+20
-19
lines changed

5 files changed

+20
-19
lines changed

docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run22.sql

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,4 +88,15 @@ USING iceberg
8888
TBLPROPERTIES(
8989
'write.format.default' = 'parquet',
9090
'format-version' = '1'
91-
);
91+
);
92+
93+
CREATE TABLE binary_partitioned_table (
94+
id BIGINT,
95+
name STRING,
96+
partition_bin BINARY
97+
)
98+
USING iceberg
99+
PARTITIONED BY (partition_bin);
100+
101+
insert into binary_partitioned_table values
102+
(1, 'a', X"0FF102FDFEFF");

fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,6 @@
117117
import java.io.IOException;
118118
import java.math.BigDecimal;
119119
import java.nio.ByteBuffer;
120-
import java.nio.charset.StandardCharsets;
121120
import java.time.DateTimeException;
122121
import java.time.Instant;
123122
import java.time.LocalDate;
@@ -686,16 +685,8 @@ private static String serializePartitionValue(org.apache.iceberg.types.Type type
686685
return null;
687686
}
688687
return value.toString();
689-
case FIXED:
690-
case BINARY:
691-
if (value == null) {
692-
return null;
693-
}
694-
// Fixed and binary types are stored as ByteBuffer
695-
ByteBuffer buffer = (ByteBuffer) value;
696-
byte[] res = new byte[buffer.limit()];
697-
buffer.get(res);
698-
return new String(res, StandardCharsets.UTF_8);
688+
// case binary, fixed should not supported, because if return string with utf8,
689+
// the data maybe be corrupted
699690
case DATE:
700691
if (value == null) {
701692
return null;

fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@
7979

8080
import java.io.FileNotFoundException;
8181
import java.io.IOException;
82-
import java.nio.charset.StandardCharsets;
8382
import java.time.DateTimeException;
8483
import java.time.LocalDate;
8584
import java.time.LocalTime;
@@ -456,12 +455,8 @@ private static String serializePartitionValue(org.apache.paimon.types.DataType t
456455
return null;
457456
}
458457
return value.toString();
459-
case BINARY:
460-
case VARBINARY:
461-
if (value == null) {
462-
return null;
463-
}
464-
return new String((byte[]) value, StandardCharsets.UTF_8);
458+
// case binary, varbinary should not supported, because if return string with utf8,
459+
// the data maybe be corrupted
465460
case DATE:
466461
if (value == null) {
467462
return null;
Binary file not shown.

regression-test/suites/external_table_p0/iceberg/test_iceberg_varbinary.groovy

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,4 +157,8 @@ suite("test_iceberg_varbinary", "p0,external,doris,external_docker,external_dock
157157
qt_select22 """
158158
select multi_distinct_count(col2),multi_distinct_count(col1) from test_ice_uuid_parquet;
159159
"""
160+
161+
qt_select23 """
162+
select * from binary_partitioned_table where from_hex(partition_bin)="0FF102FDFEFF";
163+
"""
160164
}

0 commit comments

Comments
 (0)