implement double truncation and find correct hash comparison type (#3200)

Author: jycor

enginetest/queries/join_queries.go

@@ -1200,6 +1200,37 @@ var JoinScriptTests = []ScriptTest{
 			},
 		},
 	},
+	{
+		Name: "HashLookup with type int8 and string type conversions",
+		SetUpScript: []string{
+			"create table t1 (c1 boolean);",
+			"create table t2 (c2 varchar(500));",
+			"insert into t1 values (true), (false);",
+			"insert into t2 values ('abc'), ('def');", // will be converted to float64(0) and match false
+			"insert into t2 values ('1asdf');",        // will be converted to '1' and match true
+			"insert into t2 values ('5five');",        // will be converted to '5' and match nothing
+		},
+		Assertions: []ScriptTestAssertion{
+			{
+				// TODO: our warnings don't align with MySQL
+				Query: "select /*+ HASH_JOIN(t1, t2) */ * from t1 join t2 where c1 = c2 order by c1, c2;",
+				Expected: []sql.Row{
+					{0, "abc"},
+					{0, "def"},
+					{1, "1asdf"},
+				},
+			},
+			{
+				// TODO: our warnings don't align with MySQL
+				Query: "select /*+ INNER_JOIN(t1, t2) */ * from t1 join t2 where c1 = c2 order by c1, c2;",
+				Expected: []sql.Row{
+					{0, "abc"},
+					{0, "def"},
+					{1, "1asdf"},
+				},
+			},
+		},
+	},
 }
 
 var LateralJoinScriptTests = []ScriptTest{

enginetest/queries/queries.go

@@ -892,6 +892,26 @@ var QueryTests = []QueryTest{
 		Query:    "SELECT 1 % true",
 		Expected: []sql.Row{{"0"}},
 	},
+	{
+		Query:    "select 'abc' = false",
+		Expected: []sql.Row{{true}},
+	},
+	{
+		Query:    "select '123abc' = 123",
+		Expected: []sql.Row{{true}},
+	},
+	{
+		Query:    "select '1abc' = true",
+		Expected: []sql.Row{{true}},
+	},
+	{
+		Query:    "select '123abc' = false",
+		Expected: []sql.Row{{false}},
+	},
+	{
+		Query:    "select '123abc' = true",
+		Expected: []sql.Row{{false}},
+	},
 	{
 		Query:    "SELECT * from mytable where (0.000 and true)",
 		Expected: []sql.Row{},

sql/errors.go

@@ -947,6 +947,9 @@ var (
 	ErrColumnSpecifiedTwice = errors.NewKind("column '%v' specified twice")
 
 	ErrEnumTypeTruncated = errors.NewKind("new enum type change truncates value")
+
+	// ErrTruncatedIncorrect is thrown when converting a value results in portions of the data to be trimmed.
+	ErrTruncatedIncorrect = errors.NewKind("Truncated incorrect %s value: %v")
 )
 
 // CastSQLError returns a *mysql.SQLError with the error code and in some cases, also a SQL state, populated for the

sql/expression/convert.go

@@ -372,6 +372,10 @@ func convertValue(ctx *sql.Context, val interface{}, castTo string, originType s
 		}
 		d, _, err := types.Float64.Convert(ctx, value)
 		if err != nil {
+			if sql.ErrTruncatedIncorrect.Is(err) {
+				ctx.Warn(1265, "%s", err.Error())
+				return d, nil
+			}
 			return types.Float64.Zero(), nil
 		}
 		return d, nil

sql/plan/hash_lookup.go

@@ -34,10 +34,12 @@ import (
 // simply delegates to the child.
 func NewHashLookup(n sql.Node, rightEntryKey sql.Expression, leftProbeKey sql.Expression, joinType JoinType) *HashLookup {
 	leftKeySch := hash.ExprsToSchema(leftProbeKey)
+	compareType := GetCompareType(leftProbeKey.Type(), rightEntryKey.Type())
 	return &HashLookup{
 		UnaryNode:     UnaryNode{n},
 		RightEntryKey: rightEntryKey,
 		LeftProbeKey:  leftProbeKey,
+		CompareType:   compareType,
 		Mutex:         new(sync.Mutex),
 		JoinType:      joinType,
 		leftKeySch:    leftKeySch,
@@ -48,6 +50,7 @@ type HashLookup struct {
 	UnaryNode
 	RightEntryKey sql.Expression
 	LeftProbeKey  sql.Expression
+	CompareType   sql.Type
 	Mutex         *sync.Mutex
 	Lookup        *map[interface{}][]sql.Row
 	leftKeySch    sql.Schema
@@ -58,6 +61,46 @@ var _ sql.Node = (*HashLookup)(nil)
 var _ sql.Expressioner = (*HashLookup)(nil)
 var _ sql.CollationCoercible = (*HashLookup)(nil)
 
+// GetCompareType returns the type to use when comparing values of types left and right.
+func GetCompareType(left, right sql.Type) sql.Type {
+	// TODO: much of this logic is very similar to castLeftAndRight() from sql/expression/comparison.go
+	//  consider consolidating
+	if left.Equals(right) {
+		return left
+	}
+	if types.IsTuple(left) && types.IsTuple(right) {
+		return left
+	}
+	if types.IsTime(left) || types.IsTime(right) {
+		return types.DatetimeMaxPrecision
+	}
+	if types.IsJSON(left) || types.IsJSON(right) {
+		return types.JSON
+	}
+	if types.IsBinaryType(left) || types.IsBinaryType(right) {
+		return types.LongBlob
+	}
+	if types.IsNumber(left) || types.IsNumber(right) {
+		if types.IsDecimal(left) {
+			return left
+		}
+		if types.IsDecimal(right) {
+			return right
+		}
+		if types.IsFloat(left) || types.IsFloat(right) {
+			return types.Float64
+		}
+		if types.IsSigned(left) && types.IsSigned(right) {
+			return types.Int64
+		}
+		if types.IsUnsigned(left) && types.IsUnsigned(right) {
+			return types.Uint64
+		}
+		return types.Float64
+	}
+	return types.LongText
+}
+
 func (n *HashLookup) Expressions() []sql.Expression {
 	return []sql.Expression{n.RightEntryKey, n.LeftProbeKey}
 }
@@ -113,7 +156,7 @@ func (n *HashLookup) CollationCoercibility(ctx *sql.Context) (collation sql.Coll
 	return sql.GetCoercibility(ctx, n.Child)
 }
 
-// Convert a tuple expression returning []interface{} into something comparable.
+// GetHashKey converts a tuple expression returning []interface{} into something comparable.
 // Fast paths a few smaller slices into fixed size arrays, puts everything else
 // through string serialization and a hash for now. It is OK to hash lossy here
 // as the join condition is still evaluated after the matching rows are returned.
@@ -122,13 +165,13 @@ func (n *HashLookup) GetHashKey(ctx *sql.Context, e sql.Expression, row sql.Row)
 	if err != nil {
 		return nil, err
 	}
-	typ := n.LeftProbeKey.Type()
-	key, _, err = typ.Convert(ctx, key)
+	key, _, err = n.CompareType.Convert(ctx, key)
 	if types.ErrValueNotNil.Is(err) {
 		// The LHS expression was NullType. This is allowed.
 		return nil, nil
 	}
-	if err != nil {
+	if err != nil && !sql.ErrTruncatedIncorrect.Is(err) {
+		// Truncated warning is already thrown elsewhere.
 		return nil, err
 	}
 	if s, ok := key.([]interface{}); ok {
@@ -139,7 +182,7 @@ func (n *HashLookup) GetHashKey(ctx *sql.Context, e sql.Expression, row sql.Row)
 		return string(k), nil
 	}
 
-	return hash.HashOfSimple(ctx, key, typ)
+	return hash.HashOfSimple(ctx, key, n.CompareType)
 }
 
 func (n *HashLookup) Dispose() {

sql/types/conversion.go

@@ -766,7 +766,7 @@ func TypeAwareConversion(ctx *sql.Context, val interface{}, originalType sql.Typ
 // ConvertOrTruncate converts the value |i| to type |t| and returns the converted value; if the value does not convert
 // cleanly and the type is automatically coerced (i.e. string and numeric types), then a warning is logged and the
 // value is truncated to the Zero value for type |t|. If the value does not convert and the type is not automatically
-// coerced, then an error is returned.
+// coerced, then return an error.
 func ConvertOrTruncate(ctx *sql.Context, i interface{}, t sql.Type) (interface{}, error) {
 	converted, _, err := t.Convert(ctx, i)
 	if err == nil {

sql/types/number.go

@@ -530,7 +530,7 @@ func (t NumberTypeImpl_) SQLUint64(ctx *sql.Context, dest []byte, v interface{})
 
 func (t NumberTypeImpl_) SQLFloat64(ctx *sql.Context, dest []byte, v interface{}) ([]byte, error) {
 	num, err := convertToFloat64(t, v)
-	if err != nil {
+	if err != nil && !sql.ErrTruncatedIncorrect.Is(err) {
 		return nil, err
 	}
 	dest = strconv.AppendFloat(dest, num, 'g', -1, 64)
@@ -1543,7 +1543,7 @@ func convertToFloat64(t NumberTypeImpl_, v interface{}) (float64, error) {
 			// parse the first longest valid numbers
 			s := numre.FindString(v)
 			i, _ = strconv.ParseFloat(s, 64)
-			return i, sql.ErrInvalidValue.New(v, t.String())
+			return i, sql.ErrTruncatedIncorrect.New(t.String(), v)
 		}
 		return i, nil
 	case bool:

sql/types/number_test.go

@@ -253,7 +253,7 @@ func TestNumberSQL_NumberFromString(t *testing.T) {
 
 	val, err = Float64.SQL(sql.NewEmptyContext(), nil, "also not a number")
 	require.NoError(t, err)
-	assert.Equal(t, "also not a number", val.ToString())
+	assert.Equal(t, "0", val.ToString())
 }
 
 func TestNumberString(t *testing.T) {