Skip to content

Commit 4bf2a75

Browse files
robert3005bulldozer-bot[bot]
authored andcommitted
Revert "Use SMART resolver strategy when parsing timestamps and dates (apache-spark-on-k8s#504)" (apache-spark-on-k8s#505)
Reverts palantir#504 This pr is not necessary. After more debugging it looks like it's not possible to replicate `yyyy-MM-dd HH:mm:ss.S` SDF format in java.time.
1 parent 1122b87 commit 4bf2a75

File tree

3 files changed

+3
-6
lines changed

3 files changed

+3
-6
lines changed

FORK.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
* yarn: YarnClusterSchedulerBackend, YarnSchedulerBackend
2020

2121
* [SPARK-26626](https://issues.apache.org/jira/browse/SPARK-26626) - Limited the maximum size of repeatedly substituted aliases
22-
* [SPARK-26178](https://issues.apache.org/jira/browse/SPARK-26178) (partial) - Do not use STRICT strategy when parsing timestamps
2322

2423
# Added
2524

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,6 @@ private object DateTimeFormatterHelper {
7373
.parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
7474
.toFormatter(locale)
7575
.withChronology(IsoChronology.INSTANCE)
76-
.withResolverStyle(ResolverStyle.SMART)
76+
.withResolverStyle(ResolverStyle.STRICT)
7777
}
7878
}

sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -568,10 +568,9 @@ class DateFunctionsSuite extends QueryTest with SharedSQLContext {
568568
val y1 = "2016-02-29"
569569
val y2 = "2017-02-29"
570570
val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
571-
val ts6 = Timestamp.valueOf("2017-02-28 00:00:00")
572571
val df2 = Seq(y1, y2).toDF("y")
573572
checkAnswer(df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(
574-
Row(ts5.getTime / 1000L), Row(ts6.getTime / 1000L)))
573+
Row(ts5.getTime / 1000L), Row(null)))
575574

576575
val now = sql("select unix_timestamp()").collect().head.getLong(0)
577576
checkAnswer(sql(s"select cast ($now as timestamp)"), Row(new java.util.Date(now * 1000)))
@@ -614,10 +613,9 @@ class DateFunctionsSuite extends QueryTest with SharedSQLContext {
614613
val y1 = "2016-02-29"
615614
val y2 = "2017-02-29"
616615
val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
617-
val ts6 = Timestamp.valueOf("2017-02-28 00:00:00")
618616
val df2 = Seq(y1, y2).toDF("y")
619617
checkAnswer(df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(
620-
Row(ts5.getTime / 1000L), Row(ts6.getTime / 1000L)))
618+
Row(ts5.getTime / 1000L), Row(null)))
621619

622620
// invalid format
623621
checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')"), Seq(

0 commit comments

Comments
 (0)