@@ -429,8 +429,6 @@ private[parquet] class ParquetFilters(
429
429
// Decimal type must make sure that filter value's scale matched the file.
430
430
// If doesn't matched, which would cause data corruption.
431
431
def isDecimalMatched (value : Any , decimalMeta : DecimalMetadata ): Boolean = value match {
432
- case decimalSet : Set [JBigDecimal ] =>
433
- decimalSet.iterator.next().scale == decimalMeta.getScale
434
432
case decimal : JBigDecimal =>
435
433
decimal.scale == decimalMeta.getScale
436
434
case _ => false
@@ -440,25 +438,24 @@ private[parquet] class ParquetFilters(
440
438
// in the pushed filter in order to push down the filter to Parquet.
441
439
def valueCanMakeFilterOn (name : String , value : Any ): Boolean = {
442
440
value == null || (nameToType(name) match {
443
- case ParquetBooleanType => value.isInstanceOf [JBoolean ] ||
444
- value.isInstanceOf [Set [JBoolean ]]
445
- case ParquetByteType | ParquetShortType | ParquetIntegerType =>
446
- value.isInstanceOf [Number ] || value.isInstanceOf [Set [Number ]]
447
- case ParquetLongType => value.isInstanceOf [JLong ] || value.isInstanceOf [Set [JLong ]]
448
- case ParquetFloatType => value.isInstanceOf [JFloat ] || value.isInstanceOf [Set [JFloat ]]
449
- case ParquetDoubleType => value.isInstanceOf [JDouble ] || value.isInstanceOf [Set [JDouble ]]
450
- case ParquetStringType => value.isInstanceOf [String ] || value.isInstanceOf [Set [String ]]
451
- case ParquetBinaryType => value.isInstanceOf [Array [Byte ]] ||
452
- value.isInstanceOf [Set [Array [Byte ]]]
453
- case ParquetDateType => value.isInstanceOf [Date ] || value.isInstanceOf [Set [Date ]]
441
+ case ParquetBooleanType => value.isInstanceOf [JBoolean ]
442
+ case ParquetByteType | ParquetShortType | ParquetIntegerType => value.isInstanceOf [Number ]
443
+ case ParquetLongType => value.isInstanceOf [JLong ]
444
+ case ParquetFloatType => value.isInstanceOf [JFloat ]
445
+ case ParquetDoubleType => value.isInstanceOf [JDouble ]
446
+ case ParquetStringType => value.isInstanceOf [String ]
447
+ case ParquetBinaryType => value.isInstanceOf [Array [Byte ]]
448
+ case ParquetDateType => value.isInstanceOf [Date ]
454
449
case ParquetTimestampMicrosType | ParquetTimestampMillisType =>
455
- value.isInstanceOf [Timestamp ] || value. isInstanceOf [ Set [ Timestamp ]]
450
+ value.isInstanceOf [Timestamp ]
456
451
case ParquetSchemaType (DECIMAL , INT32 , _, decimalMeta) =>
457
452
isDecimalMatched(value, decimalMeta)
458
453
case ParquetSchemaType (DECIMAL , INT64 , _, decimalMeta) =>
459
454
isDecimalMatched(value, decimalMeta)
460
455
case ParquetSchemaType (DECIMAL , FIXED_LEN_BYTE_ARRAY , _, decimalMeta) =>
461
456
isDecimalMatched(value, decimalMeta)
457
+ case set : Set [_] =>
458
+ valueCanMakeFilterOn(name, set.iterator.next())
462
459
case _ => false
463
460
})
464
461
}
0 commit comments