Skip to content

Commit 8b85a42

Browse files
committed
avoid type matching for generics
1 parent a369fc1 commit 8b85a42

File tree

1 file changed

+11
-14
lines changed

1 file changed

+11
-14
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -429,8 +429,6 @@ private[parquet] class ParquetFilters(
429429
// Decimal type must make sure that filter value's scale matched the file.
430430
// If doesn't matched, which would cause data corruption.
431431
def isDecimalMatched(value: Any, decimalMeta: DecimalMetadata): Boolean = value match {
432-
case decimalSet: Set[JBigDecimal] =>
433-
decimalSet.iterator.next().scale == decimalMeta.getScale
434432
case decimal: JBigDecimal =>
435433
decimal.scale == decimalMeta.getScale
436434
case _ => false
@@ -440,25 +438,24 @@ private[parquet] class ParquetFilters(
440438
// in the pushed filter in order to push down the filter to Parquet.
441439
def valueCanMakeFilterOn(name: String, value: Any): Boolean = {
442440
value == null || (nameToType(name) match {
443-
case ParquetBooleanType => value.isInstanceOf[JBoolean] ||
444-
value.isInstanceOf[Set[JBoolean]]
445-
case ParquetByteType | ParquetShortType | ParquetIntegerType =>
446-
value.isInstanceOf[Number] || value.isInstanceOf[Set[Number]]
447-
case ParquetLongType => value.isInstanceOf[JLong] || value.isInstanceOf[Set[JLong]]
448-
case ParquetFloatType => value.isInstanceOf[JFloat] || value.isInstanceOf[Set[JFloat]]
449-
case ParquetDoubleType => value.isInstanceOf[JDouble] || value.isInstanceOf[Set[JDouble]]
450-
case ParquetStringType => value.isInstanceOf[String] || value.isInstanceOf[Set[String]]
451-
case ParquetBinaryType => value.isInstanceOf[Array[Byte]] ||
452-
value.isInstanceOf[Set[Array[Byte]]]
453-
case ParquetDateType => value.isInstanceOf[Date] || value.isInstanceOf[Set[Date]]
441+
case ParquetBooleanType => value.isInstanceOf[JBoolean]
442+
case ParquetByteType | ParquetShortType | ParquetIntegerType => value.isInstanceOf[Number]
443+
case ParquetLongType => value.isInstanceOf[JLong]
444+
case ParquetFloatType => value.isInstanceOf[JFloat]
445+
case ParquetDoubleType => value.isInstanceOf[JDouble]
446+
case ParquetStringType => value.isInstanceOf[String]
447+
case ParquetBinaryType => value.isInstanceOf[Array[Byte]]
448+
case ParquetDateType => value.isInstanceOf[Date]
454449
case ParquetTimestampMicrosType | ParquetTimestampMillisType =>
455-
value.isInstanceOf[Timestamp] || value.isInstanceOf[Set[Timestamp]]
450+
value.isInstanceOf[Timestamp]
456451
case ParquetSchemaType(DECIMAL, INT32, _, decimalMeta) =>
457452
isDecimalMatched(value, decimalMeta)
458453
case ParquetSchemaType(DECIMAL, INT64, _, decimalMeta) =>
459454
isDecimalMatched(value, decimalMeta)
460455
case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, _, decimalMeta) =>
461456
isDecimalMatched(value, decimalMeta)
457+
case set: Set[_] =>
458+
valueCanMakeFilterOn(name, set.iterator.next())
462459
case _ => false
463460
})
464461
}

0 commit comments

Comments
 (0)