Skip to content

Commit d965217

Browse files
MaxGekkdongjoon-hyun
authored andcommitted
[SPARK-51488][SQL] Support the TIME keyword as a data type
### What changes were proposed in this pull request? In the PR, I propose to support `TIME(n)` as a data type in Spark SQL syntax, and convert it to `TimeType(precision)` during parse. ### Why are the changes needed? To achieve feature parity between Spark SQL and Scala/Java APIs. SQL users can use `TIME` as a data type. ### Does this PR introduce _any_ user-facing change? Yes, users can refer to `TIME` as a data type, but before they got an error. ### How was this patch tested? By running new tests: ``` $ build/sbt "test:testOnly *DataTypeParserSuite" ``` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #50250 from MaxGekk/time-type. Authored-by: Max Gekk <max.gekk@gmail.com> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
1 parent ee37d3a commit d965217

File tree

3 files changed

+23
-2
lines changed

3 files changed

+23
-2
lines changed

sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1280,6 +1280,7 @@ type
12801280
| FLOAT | REAL
12811281
| DOUBLE
12821282
| DATE
1283+
| TIME
12831284
| TIMESTAMP | TIMESTAMP_NTZ | TIMESTAMP_LTZ
12841285
| STRING collateClause?
12851286
| CHARACTER | CHAR

sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/DataTypeAstBuilder.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.util.SparkParserUtils.{string, withOrigin}
3030
import org.apache.spark.sql.connector.catalog.IdentityColumnSpec
3131
import org.apache.spark.sql.errors.QueryParsingErrors
3232
import org.apache.spark.sql.internal.SqlApiConf
33-
import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, CalendarIntervalType, CharType, DataType, DateType, DayTimeIntervalType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, MetadataBuilder, NullType, ShortType, StringType, StructField, StructType, TimestampNTZType, TimestampType, VarcharType, VariantType, YearMonthIntervalType}
33+
import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, CalendarIntervalType, CharType, DataType, DateType, DayTimeIntervalType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, MetadataBuilder, NullType, ShortType, StringType, StructField, StructType, TimestampNTZType, TimestampType, TimeType, VarcharType, VariantType, YearMonthIntervalType}
3434

3535
class DataTypeAstBuilder extends SqlBaseParserBaseVisitor[AnyRef] {
3636
protected def typedVisit[T](ctx: ParseTree): T = {
@@ -79,6 +79,8 @@ class DataTypeAstBuilder extends SqlBaseParserBaseVisitor[AnyRef] {
7979
case (FLOAT | REAL, Nil) => FloatType
8080
case (DOUBLE, Nil) => DoubleType
8181
case (DATE, Nil) => DateType
82+
case (TIME, Nil) => TimeType(TimeType.MICROS_PRECISION)
83+
case (TIME, precision :: Nil) => TimeType(precision.getText.toInt)
8284
case (TIMESTAMP, Nil) => SqlApiConf.get.timestampType
8385
case (TIMESTAMP_NTZ, Nil) => TimestampNTZType
8486
case (TIMESTAMP_LTZ, Nil) => TimestampType

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
package org.apache.spark.sql.catalyst.parser
1919

20-
import org.apache.spark.SparkFunSuite
20+
import org.apache.spark.{SparkException, SparkFunSuite}
2121
import org.apache.spark.sql.catalyst.plans.SQLHelper
2222
import org.apache.spark.sql.internal.SQLConf
2323
import org.apache.spark.sql.internal.SQLConf.TimestampTypes
@@ -57,6 +57,9 @@ class DataTypeParserSuite extends SparkFunSuite with SQLHelper {
5757
checkDataType("Dec(10, 5)", DecimalType(10, 5))
5858
checkDataType("deC", DecimalType.USER_DEFAULT)
5959
checkDataType("DATE", DateType)
60+
checkDataType("TimE", TimeType())
61+
checkDataType("time(0)", TimeType(0))
62+
checkDataType("TIME(6)", TimeType(6))
6063
checkDataType("timestamp", TimestampType)
6164
checkDataType("timestamp_ntz", TimestampNTZType)
6265
checkDataType("timestamp_ltz", TimestampType)
@@ -172,4 +175,19 @@ class DataTypeParserSuite extends SparkFunSuite with SQLHelper {
172175
// DataType parser accepts comments.
173176
checkDataType("Struct<x: INT, y: STRING COMMENT 'test'>",
174177
(new StructType).add("x", IntegerType).add("y", StringType, true, "test"))
178+
179+
test("unsupported precision of the time data type") {
180+
checkError(
181+
exception = intercept[SparkException] {
182+
CatalystSqlParser.parseDataType("time(9)")
183+
},
184+
condition = "UNSUPPORTED_TIME_PRECISION",
185+
parameters = Map("precision" -> "9"))
186+
checkError(
187+
exception = intercept[ParseException] {
188+
CatalystSqlParser.parseDataType("time(-1)")
189+
},
190+
condition = "PARSE_SYNTAX_ERROR",
191+
parameters = Map("error" -> "'('", "hint" -> ""))
192+
}
175193
}

0 commit comments

Comments
 (0)