Skip to content

Commit 8acc22c

Browse files
wangyumdongjoon-hyun
wangyum
authored and
dongjoon-hyun
committed
[SPARK-28138][SQL][TEST] Port timestamp.sql
## What changes were proposed in this pull request? This PR is to port timestamp.sql from PostgreSQL regression tests. https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/timestamp.sql The expected results can be found in the link: https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/expected/timestamp.out When porting the test cases, found five PostgreSQL specific features that do not exist in Spark SQL: [SPARK-28141](https://issues.apache.org/jira/browse/SPARK-28141): Timestamp type can not accept special values [SPARK-28259](https://issues.apache.org/jira/browse/SPARK-28259): Date/Time Output Styles and Date Order Conventions [SPARK-28425](https://issues.apache.org/jira/browse/SPARK-28425): Add more Date/Time Operators [SPARK-28420](https://issues.apache.org/jira/browse/SPARK-28420): Date/Time Functions: date_part [SPARK-28137](https://issues.apache.org/jira/browse/SPARK-28137): Data Type Formatting Functions [SPARK-28432](https://issues.apache.org/jira/browse/SPARK-28432): Date/Time Functions: make_date/make_timestamp Also, found one inconsistent behavior: [SPARK-27923](https://issues.apache.org/jira/browse/SPARK-27923): Spark SQL insert bad inputs to NULL ## How was this patch tested? N/A Closes apache#25181 from wangyum/SPARK-28138. Authored-by: Yuming Wang <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent 4b86510 commit 8acc22c

File tree

2 files changed

+377
-0
lines changed

2 files changed

+377
-0
lines changed

‎sql/core/src/test/resources/sql-tests/inputs/pgSQL/timestamp.sql

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
--
2+
-- Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
3+
--
4+
--
5+
-- TIMESTAMP
6+
-- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/timestamp.sql
7+
8+
CREATE TABLE TIMESTAMP_TBL (d1 timestamp) USING parquet;
9+
10+
-- [SPARK-28141] Timestamp type can not accept special values
11+
-- Test shorthand input values
12+
-- We can't just "select" the results since they aren't constants; test for
13+
-- equality instead. We can do that by running the test inside a transaction
14+
-- block, within which the value of 'now' shouldn't change. We also check
15+
-- that 'now' *does* change over a reasonable interval such as 100 msec.
16+
-- NOTE: it is possible for this part of the test to fail if the transaction
17+
-- block is entered exactly at local midnight; then 'now' and 'today' have
18+
-- the same values and the counts will come out different.
19+
20+
-- INSERT INTO TIMESTAMP_TBL VALUES ('now');
21+
-- SELECT pg_sleep(0.1);
22+
23+
-- BEGIN;
24+
25+
-- INSERT INTO TIMESTAMP_TBL VALUES ('now');
26+
-- INSERT INTO TIMESTAMP_TBL VALUES ('today');
27+
-- INSERT INTO TIMESTAMP_TBL VALUES ('yesterday');
28+
-- INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow');
29+
-- time zone should be ignored by this data type
30+
-- INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow EST');
31+
-- INSERT INTO TIMESTAMP_TBL VALUES ('tomorrow zulu');
32+
33+
-- SELECT count(*) AS One FROM TIMESTAMP_TBL WHERE d1 = timestamp 'today';
34+
-- SELECT count(*) AS Three FROM TIMESTAMP_TBL WHERE d1 = timestamp 'tomorrow';
35+
-- SELECT count(*) AS One FROM TIMESTAMP_TBL WHERE d1 = timestamp 'yesterday';
36+
-- SELECT count(*) AS One FROM TIMESTAMP_TBL WHERE d1 = timestamp(2) 'now';
37+
38+
-- COMMIT;
39+
40+
-- DELETE FROM TIMESTAMP_TBL;
41+
42+
-- verify uniform transaction time within transaction block
43+
-- BEGIN;
44+
-- INSERT INTO TIMESTAMP_TBL VALUES ('now');
45+
-- SELECT pg_sleep(0.1);
46+
-- INSERT INTO TIMESTAMP_TBL VALUES ('now');
47+
-- SELECT pg_sleep(0.1);
48+
-- SELECT count(*) AS two FROM TIMESTAMP_TBL WHERE d1 = timestamp(2) 'now';
49+
-- COMMIT;
50+
51+
-- TRUNCATE TIMESTAMP_TBL;
52+
53+
-- Special values
54+
-- INSERT INTO TIMESTAMP_TBL VALUES ('-infinity');
55+
-- INSERT INTO TIMESTAMP_TBL VALUES ('infinity');
56+
-- INSERT INTO TIMESTAMP_TBL VALUES ('epoch');
57+
-- [SPARK-27923] Spark SQL insert there obsolete special values to NULL
58+
-- Obsolete special values
59+
-- INSERT INTO TIMESTAMP_TBL VALUES ('invalid');
60+
-- INSERT INTO TIMESTAMP_TBL VALUES ('undefined');
61+
-- INSERT INTO TIMESTAMP_TBL VALUES ('current');
62+
63+
-- [SPARK-28259] Date/Time Output Styles and Date Order Conventions
64+
-- Postgres v6.0 standard output format
65+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Mon Feb 10 17:32:01 1997 PST');
66+
67+
-- Variations on Postgres v6.1 standard output format
68+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Mon Feb 10 17:32:01.000001 1997 PST');
69+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Mon Feb 10 17:32:01.999999 1997 PST');
70+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Mon Feb 10 17:32:01.4 1997 PST');
71+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Mon Feb 10 17:32:01.5 1997 PST');
72+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Mon Feb 10 17:32:01.6 1997 PST');
73+
74+
-- ISO 8601 format
75+
INSERT INTO TIMESTAMP_TBL VALUES ('1997-01-02');
76+
INSERT INTO TIMESTAMP_TBL VALUES ('1997-01-02 03:04:05');
77+
INSERT INTO TIMESTAMP_TBL VALUES ('1997-02-10 17:32:01-08');
78+
-- INSERT INTO TIMESTAMP_TBL VALUES ('1997-02-10 17:32:01-0800');
79+
-- INSERT INTO TIMESTAMP_TBL VALUES ('1997-02-10 17:32:01 -08:00');
80+
-- INSERT INTO TIMESTAMP_TBL VALUES ('19970210 173201 -0800');
81+
-- INSERT INTO TIMESTAMP_TBL VALUES ('1997-06-10 17:32:01 -07:00');
82+
INSERT INTO TIMESTAMP_TBL VALUES ('2001-09-22T18:19:20');
83+
84+
-- POSIX format (note that the timezone abbrev is just decoration here)
85+
-- INSERT INTO TIMESTAMP_TBL VALUES ('2000-03-15 08:14:01 GMT+8');
86+
-- INSERT INTO TIMESTAMP_TBL VALUES ('2000-03-15 13:14:02 GMT-1');
87+
-- INSERT INTO TIMESTAMP_TBL VALUES ('2000-03-15 12:14:03 GMT-2');
88+
-- INSERT INTO TIMESTAMP_TBL VALUES ('2000-03-15 03:14:04 PST+8');
89+
-- INSERT INTO TIMESTAMP_TBL VALUES ('2000-03-15 02:14:05 MST+7:00');
90+
91+
-- Variations for acceptable input formats
92+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 10 17:32:01 1997 -0800');
93+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 10 17:32:01 1997');
94+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 10 5:32PM 1997');
95+
-- INSERT INTO TIMESTAMP_TBL VALUES ('1997/02/10 17:32:01-0800');
96+
-- INSERT INTO TIMESTAMP_TBL VALUES ('1997-02-10 17:32:01 PST');
97+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb-10-1997 17:32:01 PST');
98+
-- INSERT INTO TIMESTAMP_TBL VALUES ('02-10-1997 17:32:01 PST');
99+
-- INSERT INTO TIMESTAMP_TBL VALUES ('19970210 173201 PST');
100+
-- set datestyle to ymd;
101+
-- INSERT INTO TIMESTAMP_TBL VALUES ('97FEB10 5:32:01PM UTC');
102+
-- INSERT INTO TIMESTAMP_TBL VALUES ('97/02/10 17:32:01 UTC');
103+
-- reset datestyle;
104+
-- INSERT INTO TIMESTAMP_TBL VALUES ('1997.041 17:32:01 UTC');
105+
-- INSERT INTO TIMESTAMP_TBL VALUES ('19970210 173201 America/New_York');
106+
-- this fails (even though TZ is a no-op, we still look it up)
107+
-- INSERT INTO TIMESTAMP_TBL VALUES ('19970710 173201 America/Does_not_exist');
108+
109+
-- Check date conversion and date arithmetic
110+
-- INSERT INTO TIMESTAMP_TBL VALUES ('1997-06-10 18:32:01 PDT');
111+
112+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 10 17:32:01 1997');
113+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 11 17:32:01 1997');
114+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 12 17:32:01 1997');
115+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 13 17:32:01 1997');
116+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 14 17:32:01 1997');
117+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 15 17:32:01 1997');
118+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 1997');
119+
120+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 0097 BC');
121+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 0097');
122+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 0597');
123+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 1097');
124+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 1697');
125+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 1797');
126+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 1897');
127+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 1997');
128+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 2097');
129+
130+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 28 17:32:01 1996');
131+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 29 17:32:01 1996');
132+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Mar 01 17:32:01 1996');
133+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Dec 30 17:32:01 1996');
134+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Dec 31 17:32:01 1996');
135+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Jan 01 17:32:01 1997');
136+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 28 17:32:01 1997');
137+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 29 17:32:01 1997');
138+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Mar 01 17:32:01 1997');
139+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Dec 30 17:32:01 1997');
140+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Dec 31 17:32:01 1997');
141+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Dec 31 17:32:01 1999');
142+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Jan 01 17:32:01 2000');
143+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Dec 31 17:32:01 2000');
144+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Jan 01 17:32:01 2001');
145+
146+
-- Currently unsupported syntax and ranges
147+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 -0097');
148+
-- INSERT INTO TIMESTAMP_TBL VALUES ('Feb 16 17:32:01 5097 BC');
149+
150+
SELECT '' AS `64`, d1 FROM TIMESTAMP_TBL;
151+
152+
-- [SPARK-28253] Date/Timestamp type have different low value and high value with Spark
153+
-- Check behavior at the lower boundary of the timestamp range
154+
-- SELECT '4714-11-24 00:00:00 BC'::timestamp;
155+
-- SELECT '4714-11-23 23:59:59 BC'::timestamp; -- out of range
156+
-- The upper boundary differs between integer and float timestamps, so no check
157+
158+
-- Demonstrate functions and operators
159+
SELECT '' AS `48`, d1 FROM TIMESTAMP_TBL
160+
WHERE d1 > timestamp '1997-01-02';
161+
162+
SELECT '' AS `15`, d1 FROM TIMESTAMP_TBL
163+
WHERE d1 < timestamp '1997-01-02';
164+
165+
SELECT '' AS one, d1 FROM TIMESTAMP_TBL
166+
WHERE d1 = timestamp '1997-01-02';
167+
168+
SELECT '' AS `63`, d1 FROM TIMESTAMP_TBL
169+
WHERE d1 != timestamp '1997-01-02';
170+
171+
SELECT '' AS `16`, d1 FROM TIMESTAMP_TBL
172+
WHERE d1 <= timestamp '1997-01-02';
173+
174+
SELECT '' AS `49`, d1 FROM TIMESTAMP_TBL
175+
WHERE d1 >= timestamp '1997-01-02';
176+
177+
-- [SPARK-28425] Add more Date/Time Operators
178+
-- SELECT '' AS `54`, d1 - timestamp '1997-01-02' AS diff
179+
-- FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
180+
181+
SELECT '' AS date_trunc_week, date_trunc( 'week', timestamp '2004-02-29 15:44:17.71393' ) AS week_trunc;
182+
183+
-- [SPARK-28425] Add more Date/Time Operators
184+
-- Test casting within a BETWEEN qualifier
185+
-- SELECT '' AS `54`, d1 - timestamp '1997-01-02' AS diff
186+
-- FROM TIMESTAMP_TBL
187+
-- WHERE d1 BETWEEN timestamp '1902-01-01'
188+
-- AND timestamp '2038-01-01';
189+
190+
-- [SPARK-28420] Date/Time Functions: date_part
191+
-- SELECT '' AS "54", d1 as "timestamp",
192+
-- date_part( 'year', d1) AS year, date_part( 'month', d1) AS month,
193+
-- date_part( 'day', d1) AS day, date_part( 'hour', d1) AS hour,
194+
-- date_part( 'minute', d1) AS minute, date_part( 'second', d1) AS second
195+
-- FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
196+
197+
-- SELECT '' AS "54", d1 as "timestamp",
198+
-- date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec,
199+
-- date_part( 'usec', d1) AS usec
200+
-- FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
201+
202+
-- SELECT '' AS "54", d1 as "timestamp",
203+
-- date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week,
204+
-- date_part( 'dow', d1) AS dow
205+
-- FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
206+
207+
-- [SPARK-28137] Data Type Formatting Functions
208+
-- TO_CHAR()
209+
-- SELECT '' AS to_char_1, to_char(d1, 'DAY Day day DY Dy dy MONTH Month month RM MON Mon mon')
210+
-- FROM TIMESTAMP_TBL;
211+
212+
-- SELECT '' AS to_char_2, to_char(d1, 'FMDAY FMDay FMday FMMONTH FMMonth FMmonth FMRM')
213+
-- FROM TIMESTAMP_TBL;
214+
215+
-- SELECT '' AS to_char_3, to_char(d1, 'Y,YYY YYYY YYY YY Y CC Q MM WW DDD DD D J')
216+
-- FROM TIMESTAMP_TBL;
217+
218+
-- SELECT '' AS to_char_4, to_char(d1, 'FMY,YYY FMYYYY FMYYY FMYY FMY FMCC FMQ FMMM FMWW FMDDD FMDD FMD FMJ')
219+
-- FROM TIMESTAMP_TBL;
220+
221+
-- SELECT '' AS to_char_5, to_char(d1, 'HH HH12 HH24 MI SS SSSS')
222+
-- FROM TIMESTAMP_TBL;
223+
224+
-- SELECT '' AS to_char_6, to_char(d1, E'"HH:MI:SS is" HH:MI:SS "\\"text between quote marks\\""')
225+
-- FROM TIMESTAMP_TBL;
226+
227+
-- SELECT '' AS to_char_7, to_char(d1, 'HH24--text--MI--text--SS')
228+
-- FROM TIMESTAMP_TBL;
229+
230+
-- SELECT '' AS to_char_8, to_char(d1, 'YYYYTH YYYYth Jth')
231+
-- FROM TIMESTAMP_TBL;
232+
233+
-- SELECT '' AS to_char_9, to_char(d1, 'YYYY A.D. YYYY a.d. YYYY bc HH:MI:SS P.M. HH:MI:SS p.m. HH:MI:SS pm')
234+
-- FROM TIMESTAMP_TBL;
235+
236+
-- SELECT '' AS to_char_10, to_char(d1, 'IYYY IYY IY I IW IDDD ID')
237+
-- FROM TIMESTAMP_TBL;
238+
239+
-- SELECT '' AS to_char_11, to_char(d1, 'FMIYYY FMIYY FMIY FMI FMIW FMIDDD FMID')
240+
-- FROM TIMESTAMP_TBL;
241+
242+
243+
--[SPARK-28432] Missing Date/Time Functions: make_timestamp
244+
-- timestamp numeric fields constructor
245+
-- SELECT make_timestamp(2014,12,28,6,30,45.887);
246+
247+
DROP TABLE TIMESTAMP_TBL;

‎sql/core/src/test/resources/sql-tests/results/pgSQL/timestamp.sql.out

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
-- Automatically generated by SQLQueryTestSuite
2+
-- Number of queries: 14
3+
4+
5+
-- !query 0
6+
CREATE TABLE TIMESTAMP_TBL (d1 timestamp) USING parquet
7+
-- !query 0 schema
8+
struct<>
9+
-- !query 0 output
10+
11+
12+
13+
-- !query 1
14+
INSERT INTO TIMESTAMP_TBL VALUES ('1997-01-02')
15+
-- !query 1 schema
16+
struct<>
17+
-- !query 1 output
18+
19+
20+
21+
-- !query 2
22+
INSERT INTO TIMESTAMP_TBL VALUES ('1997-01-02 03:04:05')
23+
-- !query 2 schema
24+
struct<>
25+
-- !query 2 output
26+
27+
28+
29+
-- !query 3
30+
INSERT INTO TIMESTAMP_TBL VALUES ('1997-02-10 17:32:01-08')
31+
-- !query 3 schema
32+
struct<>
33+
-- !query 3 output
34+
35+
36+
37+
-- !query 4
38+
INSERT INTO TIMESTAMP_TBL VALUES ('2001-09-22T18:19:20')
39+
-- !query 4 schema
40+
struct<>
41+
-- !query 4 output
42+
43+
44+
45+
-- !query 5
46+
SELECT '' AS `64`, d1 FROM TIMESTAMP_TBL
47+
-- !query 5 schema
48+
struct<64:string,d1:timestamp>
49+
-- !query 5 output
50+
1997-01-02 00:00:00
51+
1997-01-02 03:04:05
52+
1997-02-10 17:32:01
53+
2001-09-22 18:19:20
54+
55+
56+
-- !query 6
57+
SELECT '' AS `48`, d1 FROM TIMESTAMP_TBL
58+
WHERE d1 > timestamp '1997-01-02'
59+
-- !query 6 schema
60+
struct<48:string,d1:timestamp>
61+
-- !query 6 output
62+
1997-01-02 03:04:05
63+
1997-02-10 17:32:01
64+
2001-09-22 18:19:20
65+
66+
67+
-- !query 7
68+
SELECT '' AS `15`, d1 FROM TIMESTAMP_TBL
69+
WHERE d1 < timestamp '1997-01-02'
70+
-- !query 7 schema
71+
struct<15:string,d1:timestamp>
72+
-- !query 7 output
73+
74+
75+
76+
-- !query 8
77+
SELECT '' AS one, d1 FROM TIMESTAMP_TBL
78+
WHERE d1 = timestamp '1997-01-02'
79+
-- !query 8 schema
80+
struct<one:string,d1:timestamp>
81+
-- !query 8 output
82+
1997-01-02 00:00:00
83+
84+
85+
-- !query 9
86+
SELECT '' AS `63`, d1 FROM TIMESTAMP_TBL
87+
WHERE d1 != timestamp '1997-01-02'
88+
-- !query 9 schema
89+
struct<63:string,d1:timestamp>
90+
-- !query 9 output
91+
1997-01-02 03:04:05
92+
1997-02-10 17:32:01
93+
2001-09-22 18:19:20
94+
95+
96+
-- !query 10
97+
SELECT '' AS `16`, d1 FROM TIMESTAMP_TBL
98+
WHERE d1 <= timestamp '1997-01-02'
99+
-- !query 10 schema
100+
struct<16:string,d1:timestamp>
101+
-- !query 10 output
102+
1997-01-02 00:00:00
103+
104+
105+
-- !query 11
106+
SELECT '' AS `49`, d1 FROM TIMESTAMP_TBL
107+
WHERE d1 >= timestamp '1997-01-02'
108+
-- !query 11 schema
109+
struct<49:string,d1:timestamp>
110+
-- !query 11 output
111+
1997-01-02 00:00:00
112+
1997-01-02 03:04:05
113+
1997-02-10 17:32:01
114+
2001-09-22 18:19:20
115+
116+
117+
-- !query 12
118+
SELECT '' AS date_trunc_week, date_trunc( 'week', timestamp '2004-02-29 15:44:17.71393' ) AS week_trunc
119+
-- !query 12 schema
120+
struct<date_trunc_week:string,week_trunc:timestamp>
121+
-- !query 12 output
122+
2004-02-23 00:00:00
123+
124+
125+
-- !query 13
126+
DROP TABLE TIMESTAMP_TBL
127+
-- !query 13 schema
128+
struct<>
129+
-- !query 13 output
130+

0 commit comments

Comments
 (0)