Skip to content

Commit 3595364

Browse files
committed
feat: add MAP type support for first level
1 parent c5e78b6 commit 3595364

File tree

4 files changed

+13
-2
lines changed

4 files changed

+13
-2
lines changed

spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde with CometExprShim
7070
s.fields.map(_.dataType).forall(supportedDataType(_, allowComplex))
7171
case a: ArrayType if allowComplex =>
7272
supportedDataType(a.elementType, allowComplex)
73+
case m: MapType if allowComplex =>
74+
supportedDataType(m.keyType, allowComplex) && supportedDataType(m.valueType, allowComplex)
7375
case dt =>
7476
emitWarning(s"unsupported Spark data type: $dt")
7577
false

spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,10 +230,10 @@ object CometNativeScanExec extends DataTypeSupport {
230230
}
231231

232232
override def isAdditionallySupported(dt: DataType): Boolean = {
233-
// TODO add map
234233
dt match {
235234
case s: StructType => s.fields.map(_.dataType).forall(isTypeSupported)
236235
case a: ArrayType => isTypeSupported(a.elementType)
236+
case m: MapType => isTypeSupported(m.keyType) && isTypeSupported(m.valueType)
237237
case _ => false
238238
}
239239
}

spark/src/main/scala/org/apache/spark/sql/comet/CometScanExec.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -480,10 +480,10 @@ object CometScanExec extends DataTypeSupport {
480480

481481
override def isAdditionallySupported(dt: DataType): Boolean = {
482482
if (CometConf.COMET_NATIVE_SCAN_IMPL.get() == CometConf.SCAN_NATIVE_ICEBERG_COMPAT) {
483-
// TODO add map
484483
dt match {
485484
case s: StructType => s.fields.map(_.dataType).forall(isTypeSupported)
486485
case a: ArrayType => isTypeSupported(a.elementType)
486+
case m: MapType => isTypeSupported(m.keyType) && isTypeSupported(m.valueType)
487487
case _ => false
488488
}
489489
} else {

spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,4 +134,13 @@ class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper
134134
|""".stripMargin,
135135
"select c0 from tbl")
136136
}
137+
138+
test("native reader - read simple MAP fields") {
139+
testSingleLineQuery(
140+
"""
141+
|select map('a', 1) as c0 union all
142+
|select map('b', 2)
143+
|""".stripMargin,
144+
"select c0 from tbl")
145+
}
137146
}

0 commit comments

Comments
 (0)