Skip to content

Commit b9d804f

Browse files
angelamayxieangelamayxie
authored
Merge pull request #3276 from dolthub/angela/datetime
Truncate strings for datetime conversion
2 parents 679d57c + 7dd93b6 commit b9d804f

File tree

6 files changed

+188
-37
lines changed

6 files changed

+188
-37
lines changed

‎enginetest/enginetests.go‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,8 @@ func TestQueries(t *testing.T, harness Harness) {
6161
ctx := NewContext(harness)
6262
for _, tt := range queries.QueryTests {
6363
t.Run(tt.Query, func(t *testing.T) {
64-
if sh, ok := harness.(SkippingHarness); ok {
65-
if sh.SkipQueryTest(tt.Query) {
66-
t.Skipf("Skipping query plan for %s", tt.Query)
67-
}
64+
if sh, ok := harness.(SkippingHarness); tt.Skip || (ok && sh.SkipQueryTest(tt.Query)) {
65+
t.Skipf("Skipping query test for %s", tt.Query)
6866
}
6967
if IsServerEngine(e) && tt.SkipServerEngine {
7068
t.Skip("skipping for server engine")
@@ -222,7 +220,7 @@ func TestQueriesPrepared(t *testing.T, harness Harness) {
222220
defer e.Close()
223221
t.Run("query prepared tests", func(t *testing.T) {
224222
for _, tt := range queries.QueryTests {
225-
if tt.SkipPrepared {
223+
if tt.Skip || tt.SkipPrepared {
226224
continue
227225
}
228226
t.Run(tt.Query, func(t *testing.T) {
@@ -233,7 +231,7 @@ func TestQueriesPrepared(t *testing.T, harness Harness) {
233231

234232
t.Run("function query prepared tests", func(t *testing.T) {
235233
for _, tt := range queries.FunctionQueryTests {
236-
if tt.SkipPrepared {
234+
if tt.Skip || tt.SkipPrepared {
237235
continue
238236
}
239237
t.Run(tt.Query, func(t *testing.T) {
@@ -265,7 +263,7 @@ func TestQueriesPrepared(t *testing.T, harness Harness) {
265263
func TestJoinQueriesPrepared(t *testing.T, harness Harness) {
266264
harness.Setup(setup.MydbData, setup.MytableData, setup.Pk_tablesData, setup.OthertableData, setup.NiltableData, setup.XyData, setup.FooData, setup.Comp_index_tablesData)
267265
for _, tt := range queries.JoinQueryTests {
268-
if tt.SkipPrepared {
266+
if tt.Skip || tt.SkipPrepared {
269267
continue
270268
}
271269
TestPreparedQuery(t, harness, tt.Query, tt.Expected, tt.ExpectedColumns)

‎enginetest/evaluation.go‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -400,10 +400,9 @@ func TestQuery2(t *testing.T, harness Harness, e QueryEngine, q string, expected
400400
// TODO: collapse into TestQuery
401401
func TestQueryWithEngine(t *testing.T, harness Harness, e QueryEngine, tt queries.QueryTest) {
402402
t.Run(tt.Query, func(t *testing.T) {
403-
if sh, ok := harness.(SkippingHarness); ok {
404-
if sh.SkipQueryTest(tt.Query) {
405-
t.Skipf("Skipping query %s", tt.Query)
406-
}
403+
if sh, ok := harness.(SkippingHarness); tt.Skip || (IsServerEngine(e) && tt.SkipServerEngine) ||
404+
(ok && sh.SkipQueryTest(tt.Query)) {
405+
t.Skipf("Skipping query %s", tt.Query)
407406
}
408407

409408
ctx := NewContext(harness)
@@ -413,9 +412,6 @@ func TestQueryWithEngine(t *testing.T, harness Harness, e QueryEngine, tt querie
413412
} else if tt.ExpectedErrStr != "" {
414413
AssertErrWithCtx(t, e, harness, ctx, tt.Query, tt.Bindings, nil, tt.ExpectedErrStr)
415414
} else if tt.ExpectedWarning != 0 {
416-
if IsServerEngine(e) && tt.SkipServerEngine {
417-
t.Skip()
418-
}
419415
AssertWarningAndTestQuery(t, e, ctx, harness,
420416
tt.Query,
421417
tt.Expected,

‎enginetest/queries/insert_queries.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2707,6 +2707,14 @@ var InsertErrorScripts = []ScriptTest{
27072707
Query: "insert into bad values (repeat('0', 65536))",
27082708
ExpectedErr: types.ErrLengthBeyondLimit,
27092709
},
2710+
{
2711+
Name: "try inserting incorrect datetime value",
2712+
SetUpScript: []string{
2713+
"create table t (d datetime)",
2714+
},
2715+
Query: "insert into t values ('2020-01-01 a')",
2716+
ExpectedErr: sql.ErrInvalidValue,
2717+
},
27102718
}
27112719

27122720
var InsertIgnoreScripts = []ScriptTest{

‎enginetest/queries/queries.go‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ type QueryTest struct {
5050
// such as the use of the SIGNAL statement.
5151
ExpectedErrStr string
5252
// ExpectedWarning contains the expected warning code when a query generates warnings but not errors.
53+
// The ExpectedWarningsCount field must be set as well.
5354
ExpectedWarning int
5455
// ExpectedWarningsCount is used to test the expected number of warnings generated by a query.
5556
// The ExpectedWarning field must be set for warning counts to be checked.
@@ -62,6 +63,8 @@ type QueryTest struct {
6263
ExpectedColumns sql.Schema
6364
// Bindings are the bind values for the query, if provided
6465
Bindings map[string]sqlparser.Expr
66+
// Skip indicates that the query should be skipped
67+
Skip bool
6568
// SkipPrepared indicates that the query should be skipped when testing prepared statements
6669
SkipPrepared bool
6770
// SkipServerEngine indicates that the query should be skipped when testing a server engine (as opposed to the
@@ -4175,9 +4178,121 @@ SELECT * FROM cte WHERE d = 2;`,
41754178
Expected: []sql.Row{{"9999-12-31 23:59:59.999999"}},
41764179
},
41774180
{
4181+
// This returns null in MySQL, but we are able to truncate the string and convert to a datetime
4182+
Skip: true,
41784183
Query: "SELECT date_add('9999-12-31:23:59:59.99999944444444444-', INTERVAL 0 day);",
41794184
Expected: []sql.Row{{nil}},
41804185
},
4186+
// https://github.com/dolthub/dolt/issues/9917
4187+
{
4188+
Query: "select cast('2020-01-01 a' as datetime)",
4189+
ExpectedWarning: 1292,
4190+
ExpectedWarningsCount: 1,
4191+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)}},
4192+
},
4193+
{
4194+
Query: "select cast('2020-01-01 abc123' as datetime)",
4195+
ExpectedWarning: 1292,
4196+
ExpectedWarningsCount: 1,
4197+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)}},
4198+
},
4199+
{
4200+
Query: "select cast('2020-01-01 12:30asdf123' as datetime)",
4201+
ExpectedWarning: 1292,
4202+
ExpectedWarningsCount: 1,
4203+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 30, 0, 0, time.UTC)}},
4204+
},
4205+
{
4206+
Query: "select cast('2020-01-01 12:34:56abc' as datetime);",
4207+
ExpectedWarning: 1292,
4208+
ExpectedWarningsCount: 1,
4209+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
4210+
},
4211+
{
4212+
Query: "select cast('2020-01-01 12:34:56 abc' as datetime);",
4213+
ExpectedWarning: 1292,
4214+
ExpectedWarningsCount: 1, // MySQL has 2 warnings, but we don't have a warning for deprecated delimiter
4215+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
4216+
},
4217+
{
4218+
Query: "select cast('2020-01-01 12:34:56.1 abc' as datetime);",
4219+
ExpectedWarning: 1292,
4220+
ExpectedWarningsCount: 1, // MySQL has 2 warnings, but we don't have a warning for deprecated delimiter
4221+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
4222+
},
4223+
{
4224+
Query: "select cast('2020-01-01 12:34:56:123456 abc' as datetime);",
4225+
ExpectedWarning: 1292,
4226+
ExpectedWarningsCount: 1,
4227+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
4228+
},
4229+
{
4230+
Query: "select cast('2020-01-01 12:34:56...123456 abc' as datetime);",
4231+
ExpectedWarning: 1292,
4232+
ExpectedWarningsCount: 1,
4233+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
4234+
},
4235+
{
4236+
Query: "select cast('2020-01-01 12:34:56-123456 abc' as datetime);",
4237+
ExpectedWarning: 1292,
4238+
ExpectedWarningsCount: 1,
4239+
// MySQL returns null, but we are able to truncate and convert
4240+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
4241+
},
4242+
{
4243+
Query: "select cast('2020-01-01 12:34:56.123456abc' as datetime(0));",
4244+
ExpectedWarning: 1292,
4245+
ExpectedWarningsCount: 1,
4246+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
4247+
},
4248+
{
4249+
// TODO: implement precision for datetime casting
4250+
Skip: true,
4251+
Query: "select cast('2020-01-01 12:34:56.123456abc' as datetime(1));",
4252+
ExpectedWarning: 1292,
4253+
ExpectedWarningsCount: 1,
4254+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 100000000, time.UTC)}},
4255+
},
4256+
{
4257+
// TODO: implement precision for datetime casting
4258+
Skip: true,
4259+
Query: "select cast('2020-01-01 12:34:56.123456abc' as datetime(2));",
4260+
ExpectedWarning: 1292,
4261+
ExpectedWarningsCount: 1,
4262+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 120000000, time.UTC)}},
4263+
},
4264+
{
4265+
// TODO: implement precision for datetime casting
4266+
Skip: true,
4267+
Query: "select cast('2020-01-01 12:34:56.123456abc' as datetime(3));",
4268+
ExpectedWarning: 1292,
4269+
ExpectedWarningsCount: 1,
4270+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 123000000, time.UTC)}},
4271+
},
4272+
{
4273+
// TODO: implement precision for datetime casting
4274+
Skip: true,
4275+
Query: "select cast('2020-01-01 12:34:56.123456abc' as datetime(4));",
4276+
ExpectedWarning: 1292,
4277+
ExpectedWarningsCount: 1,
4278+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 123500000, time.UTC)}},
4279+
},
4280+
{
4281+
// TODO: implement precision for datetime casting
4282+
Skip: true,
4283+
Query: "select cast('2020-01-01 12:34:56.123456abc' as datetime(5));",
4284+
ExpectedWarning: 1292,
4285+
ExpectedWarningsCount: 1,
4286+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 123460000, time.UTC)}},
4287+
},
4288+
{
4289+
// TODO: implement precision for datetime casting
4290+
Skip: true,
4291+
Query: "select cast('2020-01-01 12:34:56.123456abc' as datetime(6));",
4292+
ExpectedWarning: 1292,
4293+
ExpectedWarningsCount: 1,
4294+
Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 123456000, time.UTC)}},
4295+
},
41814296
{
41824297
Query: `SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM othertable) othertable_one) othertable_two) othertable_three WHERE s2 = 'first'`,
41834298
Expected: []sql.Row{

‎sql/expression/convert.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -341,9 +341,12 @@ func convertValue(ctx *sql.Context, val interface{}, castTo string, originType s
341341
if !(isTime || isString || isBinary) {
342342
return nil, nil
343343
}
344-
d, _, err := types.DatetimeMaxPrecision.Convert(ctx, val)
344+
d, _, err := types.DatetimeDefaultPrecision.Convert(ctx, val)
345345
if err != nil {
346-
return nil, err
346+
if !sql.ErrTruncatedIncorrect.Is(err) {
347+
return nil, err
348+
}
349+
ctx.Warn(mysql.ERTruncatedWrongValue, "%s", err.Error())
347350
}
348351
return d, nil
349352
case ConvertToDecimal:

‎sql/types/datetime.go‎

Lines changed: 52 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"math"
2121
"reflect"
2222
"time"
23+
"unicode"
2324

2425
"github.com/dolthub/vitess/go/sqltypes"
2526
"github.com/dolthub/vitess/go/vt/proto/query"
@@ -33,6 +34,8 @@ const ZeroDateStr = "0000-00-00"
3334

3435
const ZeroTimestampDatetimeStr = "0000-00-00 00:00:00"
3536

37+
const MinDatetimeStringLength = 8 // length of "2000-1-1"
38+
3639
var (
3740
// ErrConvertingToTime is thrown when a value cannot be converted to a Time
3841
ErrConvertingToTime = errors.NewKind("Incorrect datetime value: '%v'")
@@ -64,31 +67,32 @@ var (
6467
datetimeMinTime = time.Date(0000, 0, 0, 0, 0, 0, 0, time.UTC)
6568

6669
DateOnlyLayouts = []string{
67-
"20060102",
68-
"2006-1-2",
6970
"2006-01-02",
7071
"2006/01/02",
72+
"20060102",
73+
"2006-1-2",
7174
}
7275

76+
TimezoneTimestampDatetimeLayout = "2006-01-02 15:04:05.999999999 -0700 MST" // represents standard Time.time.UTC()
77+
7378
// TimestampDatetimeLayouts hold extra timestamps allowed for parsing. It does
7479
// not have all the layouts supported by mysql. Missing are two digit year
7580
// versions of common cases and dates that use non common separators.
7681
//
7782
// https://github.com/MariaDB/server/blob/mysql-5.5.36/sql-common/my_time.c#L124
7883
TimestampDatetimeLayouts = append([]string{
79-
time.RFC3339,
8084
time.RFC3339Nano,
81-
"2006-01-02 15:4",
82-
"2006-01-02 15:04",
83-
"2006-01-02 15:04:",
84-
"2006-01-02 15:04:.",
85-
"2006-01-02 15:04:05.",
8685
"2006-01-02 15:04:05.999999999",
8786
"2006-1-2 15:4:5.999999999",
8887
"2006-1-2:15:4:5.999999999",
88+
time.RFC3339,
89+
"2006-01-02 15:04:05.",
8990
"2006-01-02T15:04:05",
91+
"2006-01-02 15:04:.",
92+
"2006-01-02 15:04:",
93+
"2006-01-02 15:04",
94+
"2006-01-02 15:4",
9095
"20060102150405",
91-
"2006-01-02 15:04:05.999999999 -0700 MST", // represents standard Time.time.UTC()
9296
}, DateOnlyLayouts...)
9397

9498
// zeroTime is 0000-01-01 00:00:00 UTC which is the closest Go can get to 0000-00-00 00:00:00
@@ -98,6 +102,8 @@ var (
98102
Date = MustCreateDatetimeType(sqltypes.Date, 0)
99103
// Datetime is a date and a time with default precision (no fractional seconds).
100104
Datetime = MustCreateDatetimeType(sqltypes.Datetime, 0)
105+
// DatetimeDefaultPrecision is a date and a time without a specified precision
106+
DatetimeDefaultPrecision = MustCreateDatetimeType(sqltypes.Datetime, 0)
101107
// DatetimeMaxPrecision is a date and a time with maximum precision
102108
DatetimeMaxPrecision = MustCreateDatetimeType(sqltypes.Datetime, 6)
103109
// Timestamp is a UNIX timestamp with default precision (no fractional seconds).
@@ -188,10 +194,10 @@ func (t datetimeType) Convert(ctx context.Context, v interface{}) (interface{},
188194
return nil, sql.InRange, nil
189195
}
190196
res, err := ConvertToTime(ctx, v, t)
191-
if err != nil {
197+
if err != nil && !sql.ErrTruncatedIncorrect.Is(err) {
192198
return nil, sql.OutOfRange, err
193199
}
194-
return res, sql.InRange, nil
200+
return res, sql.InRange, err
195201
}
196202

197203
// precisionConversion is a conversion ratio to divide time.Second by to truncate the appropriate amount for the
@@ -206,7 +212,7 @@ func ConvertToTime(ctx context.Context, v interface{}, t datetimeType) (time.Tim
206212
}
207213

208214
res, err := t.ConvertWithoutRangeCheck(ctx, v)
209-
if err != nil {
215+
if err != nil && !sql.ErrTruncatedIncorrect.Is(err) {
210216
return time.Time{}, err
211217
}
212218

@@ -227,7 +233,7 @@ func ConvertToTime(ctx context.Context, v interface{}, t datetimeType) (time.Tim
227233
if validated == nil {
228234
return time.Time{}, ErrConvertingToTimeOutOfRange.New(v, t)
229235
}
230-
return validated.(time.Time), nil
236+
return validated.(time.Time), err
231237
}
232238

233239
switch t.baseType {
@@ -245,7 +251,7 @@ func ConvertToTime(ctx context.Context, v interface{}, t datetimeType) (time.Tim
245251
}
246252
}
247253

248-
return res, nil
254+
return res, err
249255
}
250256

251257
// ConvertWithoutRangeCheck converts the parameter to time.Time without checking the range.
@@ -267,7 +273,7 @@ func (t datetimeType) ConvertWithoutRangeCheck(ctx context.Context, v interface{
267273
}
268274
// TODO: consider not using time.Parse if we want to match MySQL exactly ('2010-06-03 11:22.:.:.:.:' is a valid timestamp)
269275
var parsed bool
270-
res, parsed = parseDatetime(value)
276+
res, parsed, err = t.parseDatetime(value)
271277
if !parsed {
272278
return zeroTime, ErrConvertingToTime.New(v)
273279
}
@@ -361,16 +367,41 @@ func (t datetimeType) ConvertWithoutRangeCheck(ctx context.Context, v interface{
361367
res = res.Truncate(24 * time.Hour)
362368
}
363369

364-
return res, nil
370+
return res, err
371+
}
372+
373+
func (t datetimeType) parseDatetime(value string) (time.Time, bool, error) {
374+
if t, err := time.Parse(TimezoneTimestampDatetimeLayout, value); err == nil {
375+
return t.UTC(), true, nil
376+
}
377+
378+
valueLen := len(value)
379+
end := valueLen
380+
381+
for end >= MinDatetimeStringLength {
382+
for _, layout := range TimestampDatetimeLayouts {
383+
if t, err := time.Parse(layout, value[0:end]); err == nil {
384+
if end != valueLen {
385+
err = sql.ErrTruncatedIncorrect.New(t, value)
386+
}
387+
return t.UTC(), true, err
388+
}
389+
}
390+
end = findDatetimeEnd(value, end-1)
391+
}
392+
return time.Time{}, false, nil
365393
}
366394

367-
func parseDatetime(value string) (time.Time, bool) {
368-
for _, layout := range TimestampDatetimeLayouts {
369-
if t, err := time.Parse(layout, value); err == nil {
370-
return t.UTC(), true
395+
// findDatetimeEnd returns the index of the last digit before `end`
396+
func findDatetimeEnd(value string, end int) int {
397+
for end >= MinDatetimeStringLength {
398+
char := rune(value[end-1])
399+
if unicode.IsDigit(char) {
400+
return end
371401
}
402+
end--
372403
}
373-
return time.Time{}, false
404+
return end
374405
}
375406

376407
// Equals implements the Type interface.

0 commit comments

Comments
 (0)