Skip to content

Commit 03696b1

Browse files
authored
fix: [Iceberg] Fix decimal corruption (#1985)
1 parent ad873c2 commit 03696b1

File tree

1 file changed

+22
-4
lines changed

1 file changed

+22
-4
lines changed

dev/diffs/iceberg/1.8.1.diff

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ index 04ffa8f..d4107be 100644
1212
testcontainers = "1.20.4"
1313
tez010 = "0.10.4"
1414
diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle
15-
index 6eb26e8..c288e72 100644
15+
index 6eb26e8..50cefce 100644
1616
--- a/spark/v3.4/build.gradle
1717
+++ b/spark/v3.4/build.gradle
1818
@@ -75,7 +75,7 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
@@ -67,7 +67,7 @@ index 0ca1236..87daef4 100644
6767
// Controls whether reading/writing timestamps without timezones is allowed
6868
@Deprecated
6969
diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java
70-
index 4794863..8d02f02 100644
70+
index 4794863..0be31c1 100644
7171
--- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java
7272
+++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java
7373
@@ -20,11 +20,11 @@ package org.apache.iceberg.spark.data.vectorized;
@@ -83,6 +83,15 @@ index 4794863..8d02f02 100644
8383
import org.apache.comet.shaded.arrow.memory.RootAllocator;
8484
import org.apache.iceberg.parquet.VectorizedReader;
8585
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
86+
@@ -96,7 +96,7 @@ class CometColumnReader implements VectorizedReader<ColumnVector> {
87+
}
88+
89+
this.importer = new CometSchemaImporter(new RootAllocator());
90+
- this.delegate = Utils.getColumnReader(sparkType, descriptor, importer, batchSize, false, false);
91+
+ this.delegate = Utils.getColumnReader(sparkType, descriptor, importer, batchSize, true, false);
92+
this.initialized = true;
93+
}
94+
8695
diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java
8796
index a361a7f..9021cd5 100644
8897
--- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java
@@ -145,7 +154,7 @@ index 47a0e87..531b7ce 100644
145154
removeTables();
146155
sql("CREATE TABLE %s (id double, data string) USING iceberg", tableName);
147156
diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle
148-
index e2d2c7a..8b5bff8 100644
157+
index e2d2c7a..d23acef 100644
149158
--- a/spark/v3.5/build.gradle
150159
+++ b/spark/v3.5/build.gradle
151160
@@ -75,7 +75,7 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
@@ -200,7 +209,7 @@ index d6c16bb..123a300 100644
200209
public static final String CHECK_NULLABILITY = "spark.sql.iceberg.check-nullability";
201210
public static final boolean CHECK_NULLABILITY_DEFAULT = true;
202211
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java
203-
index 4794863..8d02f02 100644
212+
index 4794863..0be31c1 100644
204213
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java
205214
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java
206215
@@ -20,11 +20,11 @@ package org.apache.iceberg.spark.data.vectorized;
@@ -216,6 +225,15 @@ index 4794863..8d02f02 100644
216225
import org.apache.comet.shaded.arrow.memory.RootAllocator;
217226
import org.apache.iceberg.parquet.VectorizedReader;
218227
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
228+
@@ -96,7 +96,7 @@ class CometColumnReader implements VectorizedReader<ColumnVector> {
229+
}
230+
231+
this.importer = new CometSchemaImporter(new RootAllocator());
232+
- this.delegate = Utils.getColumnReader(sparkType, descriptor, importer, batchSize, false, false);
233+
+ this.delegate = Utils.getColumnReader(sparkType, descriptor, importer, batchSize, true, false);
234+
this.initialized = true;
235+
}
236+
219237
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java
220238
index a361a7f..9021cd5 100644
221239
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java

0 commit comments

Comments
 (0)