Truncate strings for datetime conversion

enginetest/enginetests.go

@@ -61,10 +61,8 @@ func TestQueries(t *testing.T, harness Harness) {
 	ctx := NewContext(harness)
 	for _, tt := range queries.QueryTests {
 		t.Run(tt.Query, func(t *testing.T) {
-			if sh, ok := harness.(SkippingHarness); ok {
-				if sh.SkipQueryTest(tt.Query) {
-					t.Skipf("Skipping query plan for %s", tt.Query)
-				}
+			if sh, ok := harness.(SkippingHarness); tt.Skip || (ok && sh.SkipQueryTest(tt.Query)) {
+				t.Skipf("Skipping query test for %s", tt.Query)
 			}
 			if IsServerEngine(e) && tt.SkipServerEngine {
 				t.Skip("skipping for server engine")
@@ -222,7 +220,7 @@ func TestQueriesPrepared(t *testing.T, harness Harness) {
 	defer e.Close()
 	t.Run("query prepared tests", func(t *testing.T) {
 		for _, tt := range queries.QueryTests {
-			if tt.SkipPrepared {
+			if tt.Skip || tt.SkipPrepared {
 				continue
 			}
 			t.Run(tt.Query, func(t *testing.T) {
@@ -233,7 +231,7 @@ func TestQueriesPrepared(t *testing.T, harness Harness) {
 
 	t.Run("function query prepared tests", func(t *testing.T) {
 		for _, tt := range queries.FunctionQueryTests {
-			if tt.SkipPrepared {
+			if tt.Skip || tt.SkipPrepared {
 				continue
 			}
 			t.Run(tt.Query, func(t *testing.T) {
@@ -265,7 +263,7 @@ func TestQueriesPrepared(t *testing.T, harness Harness) {
 func TestJoinQueriesPrepared(t *testing.T, harness Harness) {
 	harness.Setup(setup.MydbData, setup.MytableData, setup.Pk_tablesData, setup.OthertableData, setup.NiltableData, setup.XyData, setup.FooData, setup.Comp_index_tablesData)
 	for _, tt := range queries.JoinQueryTests {
-		if tt.SkipPrepared {
+		if tt.Skip || tt.SkipPrepared {
 			continue
 		}
 		TestPreparedQuery(t, harness, tt.Query, tt.Expected, tt.ExpectedColumns)

enginetest/evaluation.go

@@ -400,10 +400,9 @@ func TestQuery2(t *testing.T, harness Harness, e QueryEngine, q string, expected
 // TODO: collapse into TestQuery
 func TestQueryWithEngine(t *testing.T, harness Harness, e QueryEngine, tt queries.QueryTest) {
 	t.Run(tt.Query, func(t *testing.T) {
-		if sh, ok := harness.(SkippingHarness); ok {
-			if sh.SkipQueryTest(tt.Query) {
-				t.Skipf("Skipping query %s", tt.Query)
-			}
+		if sh, ok := harness.(SkippingHarness); tt.Skip || (IsServerEngine(e) && tt.SkipServerEngine) ||
+			(ok && sh.SkipQueryTest(tt.Query)) {
+			t.Skipf("Skipping query %s", tt.Query)
 		}
 
 		ctx := NewContext(harness)
@@ -413,9 +412,6 @@ func TestQueryWithEngine(t *testing.T, harness Harness, e QueryEngine, tt querie
 		} else if tt.ExpectedErrStr != "" {
 			AssertErrWithCtx(t, e, harness, ctx, tt.Query, tt.Bindings, nil, tt.ExpectedErrStr)
 		} else if tt.ExpectedWarning != 0 {
-			if IsServerEngine(e) && tt.SkipServerEngine {
-				t.Skip()
-			}
 			AssertWarningAndTestQuery(t, e, ctx, harness,
 				tt.Query,
 				tt.Expected,

enginetest/queries/insert_queries.go

@@ -2707,6 +2707,14 @@ var InsertErrorScripts = []ScriptTest{
 		Query:       "insert into bad values (repeat('0', 65536))",
 		ExpectedErr: types.ErrLengthBeyondLimit,
 	},
+	{
+		Name: "try inserting incorrect datetime value",
+		SetUpScript: []string{
+			"create table t (d datetime)",
+		},
+		Query:       "insert into t values ('2020-01-01 a')",
+		ExpectedErr: sql.ErrInvalidValue,
+	},
 }
 
 var InsertIgnoreScripts = []ScriptTest{

enginetest/queries/queries.go

@@ -50,6 +50,7 @@ type QueryTest struct {
 	// such as the use of the SIGNAL statement.
 	ExpectedErrStr string
 	// ExpectedWarning contains the expected warning code when a query generates warnings but not errors.
+	// The ExpectedWarningsCount field must be set as well.
 	ExpectedWarning int
 	// ExpectedWarningsCount is used to test the expected number of warnings generated by a query.
 	// The ExpectedWarning field must be set for warning counts to be checked.
@@ -62,6 +63,8 @@ type QueryTest struct {
 	ExpectedColumns sql.Schema
 	// Bindings are the bind values for the query, if provided
 	Bindings map[string]sqlparser.Expr
+	// Skip indicates that the query should be skipped
+	Skip bool
 	// SkipPrepared indicates that the query should be skipped when testing prepared statements
 	SkipPrepared bool
 	// SkipServerEngine indicates that the query should be skipped when testing a server engine (as opposed to the
@@ -4175,9 +4178,121 @@ SELECT * FROM cte WHERE  d = 2;`,
 		Expected: []sql.Row{{"9999-12-31 23:59:59.999999"}},
 	},
 	{
+		// This returns null in MySQL, but we are able to truncate the string and convert to a datetime
+		Skip:     true,
 		Query:    "SELECT date_add('9999-12-31:23:59:59.99999944444444444-', INTERVAL 0 day);",
 		Expected: []sql.Row{{nil}},
 	},
+	// https://github.com/dolthub/dolt/issues/9917
+	{
+		Query:                 "select cast('2020-01-01 a' as datetime)",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)}},
+	},
+	{
+		Query:                 "select cast('2020-01-01 abc123' as datetime)",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)}},
+	},
+	{
+		Query:                 "select cast('2020-01-01 12:30asdf123' as datetime)",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 30, 0, 0, time.UTC)}},
+	},
+	{
+		Query:                 "select cast('2020-01-01 12:34:56abc' as datetime);",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
+	},
+	{
+		Query:                 "select cast('2020-01-01 12:34:56 abc' as datetime);",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1, // MySQL has 2 warnings, but we don't have a warning for deprecated delimiter
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
+	},
+	{
+		Query:                 "select cast('2020-01-01 12:34:56.1 abc' as datetime);",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1, // MySQL has 2 warnings, but we don't have a warning for deprecated delimiter
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
+	},
+	{
+		Query:                 "select cast('2020-01-01 12:34:56:123456 abc' as datetime);",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
+	},
+	{
+		Query:                 "select cast('2020-01-01 12:34:56...123456 abc' as datetime);",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
+	},
+	{
+		Query:                 "select cast('2020-01-01 12:34:56-123456 abc' as datetime);",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		// MySQL returns null, but we are able to truncate and convert
+		Expected: []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
+	},
+	{
+		Query:                 "select cast('2020-01-01 12:34:56.123456abc' as datetime(0));",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 0, time.UTC)}},
+	},
+	{
+		// TODO: implement precision for datetime casting
+		Skip:                  true,
+		Query:                 "select cast('2020-01-01 12:34:56.123456abc' as datetime(1));",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 100000000, time.UTC)}},
+	},
+	{
+		// TODO: implement precision for datetime casting
+		Skip:                  true,
+		Query:                 "select cast('2020-01-01 12:34:56.123456abc' as datetime(2));",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 120000000, time.UTC)}},
+	},
+	{
+		// TODO: implement precision for datetime casting
+		Skip:                  true,
+		Query:                 "select cast('2020-01-01 12:34:56.123456abc' as datetime(3));",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 123000000, time.UTC)}},
+	},
+	{
+		// TODO: implement precision for datetime casting
+		Skip:                  true,
+		Query:                 "select cast('2020-01-01 12:34:56.123456abc' as datetime(4));",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 123500000, time.UTC)}},
+	},
+	{
+		// TODO: implement precision for datetime casting
+		Skip:                  true,
+		Query:                 "select cast('2020-01-01 12:34:56.123456abc' as datetime(5));",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 123460000, time.UTC)}},
+	},
+	{
+		// TODO: implement precision for datetime casting
+		Skip:                  true,
+		Query:                 "select cast('2020-01-01 12:34:56.123456abc' as datetime(6));",
+		ExpectedWarning:       1292,
+		ExpectedWarningsCount: 1,
+		Expected:              []sql.Row{{time.Date(2020, time.January, 1, 12, 34, 56, 123456000, time.UTC)}},
+	},
 	{
 		Query: `SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM othertable) othertable_one) othertable_two) othertable_three WHERE s2 = 'first'`,
 		Expected: []sql.Row{

sql/expression/convert.go

@@ -341,9 +341,12 @@ func convertValue(ctx *sql.Context, val interface{}, castTo string, originType s
 		if !(isTime || isString || isBinary) {
 			return nil, nil
 		}
-		d, _, err := types.DatetimeMaxPrecision.Convert(ctx, val)
+		d, _, err := types.DatetimeDefaultPrecision.Convert(ctx, val)
 		if err != nil {
-			return nil, err
+			if !sql.ErrTruncatedIncorrect.Is(err) {
+				return nil, err
+			}
+			ctx.Warn(mysql.ERTruncatedWrongValue, "%s", err.Error())
 		}
 		return d, nil
 	case ConvertToDecimal:

sql/types/datetime.go

@@ -20,6 +20,7 @@ import (
 	"math"
 	"reflect"
 	"time"
+	"unicode"
 
 	"github.com/dolthub/vitess/go/sqltypes"
 	"github.com/dolthub/vitess/go/vt/proto/query"
@@ -33,6 +34,8 @@ const ZeroDateStr = "0000-00-00"
 
 const ZeroTimestampDatetimeStr = "0000-00-00 00:00:00"
 
+const MinDatetimeStringLength = 8 // length of "2000-1-1"
+
 var (
 	// ErrConvertingToTime is thrown when a value cannot be converted to a Time
 	ErrConvertingToTime = errors.NewKind("Incorrect datetime value: '%v'")
@@ -64,31 +67,32 @@ var (
 	datetimeMinTime = time.Date(0000, 0, 0, 0, 0, 0, 0, time.UTC)
 
 	DateOnlyLayouts = []string{
-		"20060102",
-		"2006-1-2",
 		"2006-01-02",
 		"2006/01/02",
+		"20060102",
+		"2006-1-2",
 	}
 
+	TimezoneTimestampDatetimeLayout = "2006-01-02 15:04:05.999999999 -0700 MST" // represents standard Time.time.UTC()
+
 	// TimestampDatetimeLayouts hold extra timestamps allowed for parsing. It does
 	// not have all the layouts supported by mysql. Missing are two digit year
 	// versions of common cases and dates that use non common separators.
 	//
 	// https://github.com/MariaDB/server/blob/mysql-5.5.36/sql-common/my_time.c#L124
 	TimestampDatetimeLayouts = append([]string{
-		time.RFC3339,
 		time.RFC3339Nano,
-		"2006-01-02 15:4",
-		"2006-01-02 15:04",
-		"2006-01-02 15:04:",
-		"2006-01-02 15:04:.",
-		"2006-01-02 15:04:05.",
 		"2006-01-02 15:04:05.999999999",
 		"2006-1-2 15:4:5.999999999",
 		"2006-1-2:15:4:5.999999999",
+		time.RFC3339,
+		"2006-01-02 15:04:05.",
 		"2006-01-02T15:04:05",
+		"2006-01-02 15:04:.",
+		"2006-01-02 15:04:",
+		"2006-01-02 15:04",
+		"2006-01-02 15:4",
 		"20060102150405",
-		"2006-01-02 15:04:05.999999999 -0700 MST", // represents standard Time.time.UTC()
 	}, DateOnlyLayouts...)
 
 	// zeroTime is 0000-01-01 00:00:00 UTC which is the closest Go can get to 0000-00-00 00:00:00
@@ -98,6 +102,8 @@ var (
 	Date = MustCreateDatetimeType(sqltypes.Date, 0)
 	// Datetime is a date and a time with default precision (no fractional seconds).
 	Datetime = MustCreateDatetimeType(sqltypes.Datetime, 0)
+	// DatetimeDefaultPrecision is a date and a time without a specified precision
+	DatetimeDefaultPrecision = MustCreateDatetimeType(sqltypes.Datetime, 0)
 	// DatetimeMaxPrecision is a date and a time with maximum precision
 	DatetimeMaxPrecision = MustCreateDatetimeType(sqltypes.Datetime, 6)
 	// Timestamp is a UNIX timestamp with default precision (no fractional seconds).
@@ -188,10 +194,10 @@ func (t datetimeType) Convert(ctx context.Context, v interface{}) (interface{},
 		return nil, sql.InRange, nil
 	}
 	res, err := ConvertToTime(ctx, v, t)
-	if err != nil {
+	if err != nil && !sql.ErrTruncatedIncorrect.Is(err) {
 		return nil, sql.OutOfRange, err
 	}
-	return res, sql.InRange, nil
+	return res, sql.InRange, err
 }
 
 // precisionConversion is a conversion ratio to divide time.Second by to truncate the appropriate amount for the
@@ -206,7 +212,7 @@ func ConvertToTime(ctx context.Context, v interface{}, t datetimeType) (time.Tim
 	}
 
 	res, err := t.ConvertWithoutRangeCheck(ctx, v)
-	if err != nil {
+	if err != nil && !sql.ErrTruncatedIncorrect.Is(err) {
 		return time.Time{}, err
 	}
 
@@ -227,7 +233,7 @@ func ConvertToTime(ctx context.Context, v interface{}, t datetimeType) (time.Tim
 		if validated == nil {
 			return time.Time{}, ErrConvertingToTimeOutOfRange.New(v, t)
 		}
-		return validated.(time.Time), nil
+		return validated.(time.Time), err
 	}
 
 	switch t.baseType {
@@ -245,7 +251,7 @@ func ConvertToTime(ctx context.Context, v interface{}, t datetimeType) (time.Tim
 		}
 	}
 
-	return res, nil
+	return res, err
 }
 
 // ConvertWithoutRangeCheck converts the parameter to time.Time without checking the range.
@@ -267,7 +273,7 @@ func (t datetimeType) ConvertWithoutRangeCheck(ctx context.Context, v interface{
 		}
 		// TODO: consider not using time.Parse if we want to match MySQL exactly ('2010-06-03 11:22.:.:.:.:' is a valid timestamp)
 		var parsed bool
-		res, parsed = parseDatetime(value)
+		res, parsed, err = t.parseDatetime(value)
 		if !parsed {
 			return zeroTime, ErrConvertingToTime.New(v)
 		}
@@ -361,16 +367,41 @@ func (t datetimeType) ConvertWithoutRangeCheck(ctx context.Context, v interface{
 		res = res.Truncate(24 * time.Hour)
 	}
 
-	return res, nil
+	return res, err
+}
+
+func (t datetimeType) parseDatetime(value string) (time.Time, bool, error) {
+	if t, err := time.Parse(TimezoneTimestampDatetimeLayout, value); err == nil {
+		return t.UTC(), true, nil
+	}
+
+	valueLen := len(value)
+	end := valueLen
+
+	for end >= MinDatetimeStringLength {
+		for _, layout := range TimestampDatetimeLayouts {
+			if t, err := time.Parse(layout, value[0:end]); err == nil {
+				if end != valueLen {
+					err = sql.ErrTruncatedIncorrect.New(t, value)
+				}
+				return t.UTC(), true, err
+			}
+		}
+		end = findDatetimeEnd(value, end-1)
+	}
+	return time.Time{}, false, nil
 }
 
-func parseDatetime(value string) (time.Time, bool) {
-	for _, layout := range TimestampDatetimeLayouts {
-		if t, err := time.Parse(layout, value); err == nil {
-			return t.UTC(), true
+// findDatetimeEnd returns the index of the last digit before `end`
+func findDatetimeEnd(value string, end int) int {
+	for end >= MinDatetimeStringLength {
+		char := rune(value[end-1])
+		if unicode.IsDigit(char) {
+			return end
 		}
+		end--
 	}
-	return time.Time{}, false
+	return end
 }
 
 // Equals implements the Type interface.