Skip to content

Commit dc38be5

Browse files
Catalin TodaCatalin Toda
authored andcommitted
Test fixing Long type in theschema and int32 parquet type
1 parent c94d53a commit dc38be5

File tree

1 file changed

+2
-5
lines changed

1 file changed

+2
-5
lines changed

sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorUpdaterFactory.java

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,8 @@ public ParquetVectorUpdater getUpdater(ColumnDescriptor descriptor, DataType spa
7272
case INT32:
7373
if (sparkType == DataTypes.IntegerType || canReadAsIntDecimal(descriptor, sparkType)) {
7474
return new IntegerUpdater();
75-
} else if (sparkType == DataTypes.LongType && isUnsignedIntTypeMatched(32)) {
76-
// In `ParquetToSparkSchemaConverter`, we map parquet UINT32 to our LongType.
77-
// For unsigned int32, it stores as plain signed int32 in Parquet when dictionary
78-
// fallbacks. We read them as long values.
79-
return new UnsignedIntegerUpdater();
75+
} else if (sparkType == DataTypes.LongType) {
76+
return new IntegerUpdater();
8077
} else if (sparkType == DataTypes.ByteType) {
8178
return new ByteUpdater();
8279
} else if (sparkType == DataTypes.ShortType) {

0 commit comments

Comments
 (0)