Merge pull request #2967 from dolthub/nicktobey/hash

nicktobey · web-flow · commit 9a519d19efa7 · 2025-05-05T17:28:37.000-07:00
Unwrap wrapped values before computing their hash.
diff --git a/enginetest/enginetests.go b/enginetest/enginetests.go
@@ -3046,7 +3046,7 @@ func TestRenameColumn(t *testing.T, harness Harness) {
 		TestQueryWithContext(t, ctx, e, harness, "ALTER TABLE mydb.tabletest RENAME COLUMN s TO i1", []sql.Row{{types.NewOkResult(0)}}, nil, nil, nil)
 		TestQueryWithContext(t, ctx, e, harness, "SHOW FULL COLUMNS FROM mydb.tabletest", []sql.Row{
 			{"i", "int", nil, "NO", "PRI", nil, "", "", ""},
-			{"i1", "varchar(20)", "utf8mb4_0900_bin", "NO", "", nil, "", "", ""},
+			{"i1", "text", "utf8mb4_0900_bin", "NO", "", nil, "", "", ""},
 		}, nil, nil, nil)
 	})
 }
diff --git a/enginetest/join_planning_tests.go b/enginetest/join_planning_tests.go
@@ -945,6 +945,32 @@ where u in (select * from rec);`,
 			},
 		},
 	},
+	{
+		name: "join varchar and text columns",
+		setup: []string{
+			"CREATE table varchartable (pk int primary key, s varchar(20));",
+			"CREATE table texttable (pk int primary key, t text);",
+			"insert into varchartable values (1,'first'), (2,'second'), (3,'third');",
+			"insert into texttable values (1,'first'), (2,'second'), (3,'third');",
+		},
+		// write a bunch of left joins and make sure they are converted to anti joins
+		tests: []JoinPlanTest{
+			{
+				q:     "select /*+ HASH_JOIN(varchartable,texttable) */ * from varchartable where s in (select t from texttable) order by pk",
+				types: []plan.JoinType{plan.JoinTypeHash},
+				exp: []sql.Row{
+					{1, "first"},
+					{2, "second"},
+					{3, "third"},
+				},
+			},
+			{
+				q:     "select /*+ HASH_JOIN(varchartable,texttable) */ * from varchartable where s not in (select t from texttable) order by pk",
+				types: []plan.JoinType{plan.JoinTypeLeftOuterHashExcludeNulls},
+				exp:   []sql.Row{},
+			},
+		},
+	},
 	{
 		name: "join concat tests",
 		setup: []string{
diff --git a/enginetest/queries/queries.go b/enginetest/queries/queries.go
@@ -5665,6 +5665,18 @@ SELECT * FROM cte WHERE  d = 2;`,
 			{"third row"},
 		},
 	},
+	{
+		Query:    "select * from mytable intersect select * from tabletest",
+		Expected: []sql.Row{{1, "first row"}, {2, "second row"}, {3, "third row"}},
+	},
+	{
+		Query:    "select * from mytable union distinct select * from tabletest",
+		Expected: []sql.Row{{1, "first row"}, {2, "second row"}, {3, "third row"}},
+	},
+	{
+		Query:    "select * from mytable except select * from tabletest",
+		Expected: []sql.Row{},
+	},
 	{
 		SkipPrepared: true,
 		Query:        "",
@@ -6772,6 +6784,10 @@ SELECT * FROM cte WHERE  d = 2;`,
 		Query:    "select replace(s, 'row', '') from mytable order by i",
 		Expected: []sql.Row{{"first "}, {"second "}, {"third "}},
 	},
+	{
+		Query:    "select replace(s, 'row', '') from tabletest order by i",
+		Expected: []sql.Row{{"first "}, {"second "}, {"third "}},
+	},
 	{
 		Query:    "select rpad(s, 13, ' ') from mytable order by i",
 		Expected: []sql.Row{{"first row    "}, {"second row   "}, {"third row    "}},
@@ -6780,6 +6796,10 @@ SELECT * FROM cte WHERE  d = 2;`,
 		Query:    "select lpad(s, 13, ' ') from mytable order by i",
 		Expected: []sql.Row{{"    first row"}, {"   second row"}, {"    third row"}},
 	},
+	{
+		Query:    "select lpad(s, 13, ' ') from tabletest order by i",
+		Expected: []sql.Row{{"    first row"}, {"   second row"}, {"    third row"}},
+	},
 	{
 		Query:    "select sqrt(i) from mytable order by i",
 		Expected: []sql.Row{{1.0}, {1.4142135623730951}, {1.7320508075688772}},
diff --git a/enginetest/scriptgen/setup/scripts/tabletest b/enginetest/scriptgen/setup/scripts/tabletest
@@ -1,7 +1,7 @@
 exec
 create table tabletest (
     i int primary key,
-    s varchar(20) not null
+    s text not null
 )
 ----
 
diff --git a/enginetest/scriptgen/setup/setup_data.sg.go b/enginetest/scriptgen/setup/setup_data.sg.go
diff --git a/memory/table.go b/memory/table.go
@@ -1971,7 +1971,7 @@ func (t *Table) DropCheck(ctx *sql.Context, chName string) error {
 	return fmt.Errorf("check '%s' was not found on the table", chName)
 }
 
-func (t *Table) createIndex(data *TableData, name string, columns []sql.IndexColumn, constraint sql.IndexConstraint, comment string) (sql.Index, error) {
+func (t *Table) createIndex(ctx *sql.Context, data *TableData, name string, columns []sql.IndexColumn, constraint sql.IndexConstraint, comment string) (sql.Index, error) {
 	if name == "" {
 		for _, column := range columns {
 			name += column.Name + "_"
@@ -2005,7 +2005,7 @@ func (t *Table) createIndex(data *TableData, name string, columns []sql.IndexCol
 	}
 
 	if constraint == sql.IndexConstraint_Unique {
-		err := data.errIfDuplicateEntryExist(colNames, name)
+		err := data.errIfDuplicateEntryExist(ctx, colNames, name)
 		if err != nil {
 			return nil, err
 		}
@@ -2041,7 +2041,7 @@ func (t *Table) CreateIndex(ctx *sql.Context, idx sql.IndexDef) error {
 		data.indexes = make(map[string]sql.Index)
 	}
 
-	index, err := t.createIndex(data, idx.Name, idx.Columns, idx.Constraint, idx.Comment)
+	index, err := t.createIndex(ctx, data, idx.Name, idx.Columns, idx.Constraint, idx.Comment)
 	if err != nil {
 		return err
 	}
@@ -2107,7 +2107,7 @@ func (t *Table) CreateFulltextIndex(ctx *sql.Context, indexDef sql.IndexDef, key
 		data.indexes = make(map[string]sql.Index)
 	}
 
-	index, err := t.createIndex(data, indexDef.Name, indexDef.Columns, indexDef.Constraint, indexDef.Comment)
+	index, err := t.createIndex(ctx, data, indexDef.Name, indexDef.Columns, indexDef.Constraint, indexDef.Comment)
 	if err != nil {
 		return err
 	}
@@ -2138,7 +2138,7 @@ func (t *Table) CreateVectorIndex(ctx *sql.Context, idx sql.IndexDef, distanceTy
 		data.indexes = make(map[string]sql.Index)
 	}
 
-	index, err := t.createIndex(data, idx.Name, idx.Columns, idx.Constraint, idx.Comment)
+	index, err := t.createIndex(ctx, data, idx.Name, idx.Columns, idx.Constraint, idx.Comment)
 	if err != nil {
 		return err
 	}
diff --git a/memory/table_data.go b/memory/table_data.go
@@ -15,6 +15,7 @@
 package memory
 
 import (
+	"context"
 	"fmt"
 	"sort"
 	"strconv"
@@ -274,7 +275,7 @@ func (td *TableData) numRows(ctx *sql.Context) (uint64, error) {
 }
 
 // throws an error if any two or more rows share the same |cols| values.
-func (td *TableData) errIfDuplicateEntryExist(cols []string, idxName string) error {
+func (td *TableData) errIfDuplicateEntryExist(ctx context.Context, cols []string, idxName string) error {
 	columnMapping, err := td.columnIndexes(cols)
 
 	// We currently skip validating duplicates on unique virtual columns.
@@ -296,7 +297,7 @@ func (td *TableData) errIfDuplicateEntryExist(cols []string, idxName string) err
 			if hasNulls(idxPrefixKey) {
 				continue
 			}
-			h, err := sql.HashOf(idxPrefixKey)
+			h, err := sql.HashOf(ctx, idxPrefixKey)
 			if err != nil {
 				return err
 			}
diff --git a/sql/cache.go b/sql/cache.go
@@ -15,6 +15,7 @@
 package sql
 
 import (
+	"context"
 	"fmt"
 	"runtime"
 	"sync"
@@ -25,7 +26,7 @@ import (
 )
 
 // HashOf returns a hash of the given value to be used as key in a cache.
-func HashOf(v Row) (uint64, error) {
+func HashOf(ctx context.Context, v Row) (uint64, error) {
 	hash := digestPool.Get().(*xxhash.Digest)
 	hash.Reset()
 	defer digestPool.Put(hash)
@@ -36,7 +37,10 @@ func HashOf(v Row) (uint64, error) {
 				return 0, err
 			}
 		}
-
+		x, err := UnwrapAny(ctx, x)
+		if err != nil {
+			return 0, err
+		}
 		// TODO: probably much faster to do this with a type switch
 		// TODO: we don't have the type info necessary to appropriately encode the value of a string with a non-standard
 		//  collation, which means that two strings that differ only in their collations will hash to the same value.
diff --git a/sql/cache_test.go b/sql/cache_test.go
@@ -15,6 +15,7 @@
 package sql
 
 import (
+	"context"
 	"errors"
 	"testing"
 
@@ -179,10 +180,11 @@ func TestRowsCache(t *testing.T) {
 }
 
 func BenchmarkHashOf(b *testing.B) {
+	ctx := context.Background()
 	row := NewRow(1, "1")
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		sum, err := HashOf(row)
+		sum, err := HashOf(ctx, row)
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -193,11 +195,12 @@ func BenchmarkHashOf(b *testing.B) {
 }
 
 func BenchmarkParallelHashOf(b *testing.B) {
+	ctx := context.Background()
 	row := NewRow(1, "1")
 	b.ResetTimer()
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			sum, err := HashOf(row)
+			sum, err := HashOf(ctx, row)
 			if err != nil {
 				b.Fatal(err)
 			}
diff --git a/sql/iters/rel_iters.go b/sql/iters/rel_iters.go
@@ -571,7 +571,7 @@ func (di *distinctIter) Next(ctx *sql.Context) (sql.Row, error) {
 			return nil, err
 		}
 
-		hash, err := sql.HashOf(row)
+		hash, err := sql.HashOf(ctx, row)
 		if err != nil {
 			return nil, err
 		}
@@ -647,7 +647,7 @@ func (ii *IntersectIter) Next(ctx *sql.Context) (sql.Row, error) {
 				return nil, err
 			}
 
-			hash, herr := sql.HashOf(res)
+			hash, herr := sql.HashOf(ctx, res)
 			if herr != nil {
 				return nil, herr
 			}
@@ -669,7 +669,7 @@ func (ii *IntersectIter) Next(ctx *sql.Context) (sql.Row, error) {
 			return nil, err
 		}
 
-		hash, herr := sql.HashOf(res)
+		hash, herr := sql.HashOf(ctx, res)
 		if herr != nil {
 			return nil, herr
 		}
@@ -714,7 +714,7 @@ func (ei *ExceptIter) Next(ctx *sql.Context) (sql.Row, error) {
 				return nil, err
 			}
 
-			hash, herr := sql.HashOf(res)
+			hash, herr := sql.HashOf(ctx, res)
 			if herr != nil {
 				return nil, herr
 			}
@@ -736,7 +736,7 @@ func (ei *ExceptIter) Next(ctx *sql.Context) (sql.Row, error) {
 			return nil, err
 		}
 
-		hash, herr := sql.HashOf(res)
+		hash, herr := sql.HashOf(ctx, res)
 		if herr != nil {
 			return nil, herr
 		}
diff --git a/sql/plan/hash_lookup.go b/sql/plan/hash_lookup.go
@@ -127,7 +127,7 @@ func (n *HashLookup) GetHashKey(ctx *sql.Context, e sql.Expression, row sql.Row)
 		return nil, err
 	}
 	if s, ok := key.([]interface{}); ok {
-		return sql.HashOf(s)
+		return sql.HashOf(ctx, s)
 	}
 	// byte slices are not hashable
 	if k, ok := key.([]byte); ok {
diff --git a/sql/plan/insubquery.go b/sql/plan/insubquery.go
@@ -47,7 +47,7 @@ func NewInSubquery(left sql.Expression, right sql.Expression) *InSubquery {
 	return &InSubquery{expression.BinaryExpressionStub{LeftChild: left, RightChild: right}}
 }
 
-var nilKey, _ = sql.HashOf(sql.NewRow(nil))
+var nilKey, _ = sql.HashOf(nil, sql.NewRow(nil))
 
 // Eval implements the Expression interface.
 func (in *InSubquery) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
@@ -96,7 +96,7 @@ func (in *InSubquery) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
 			return false, nil
 		}
 
-		key, err := sql.HashOf(sql.NewRow(nLeft))
+		key, err := sql.HashOf(ctx, sql.NewRow(nLeft))
 		if err != nil {
 			return nil, err
 		}
diff --git a/sql/plan/subquery.go b/sql/plan/subquery.go
@@ -477,7 +477,7 @@ func putAllRows(ctx *sql.Context, cache sql.KeyValueCache, vals []interface{}) e
 		if err != nil {
 			return err
 		}
-		rowKey, err := sql.HashOf(sql.NewRow(val))
+		rowKey, err := sql.HashOf(ctx, sql.NewRow(val))
 		if err != nil {
 			return err
 		}
diff --git a/sql/rowexec/join_iters.go b/sql/rowexec/join_iters.go
@@ -462,7 +462,7 @@ func (i *fullJoinIter) Next(ctx *sql.Context) (sql.Row, error) {
 
 		rightRow, err := i.r.Next(ctx)
 		if err == io.EOF {
-			key, err := sql.HashOf(i.leftRow)
+			key, err := sql.HashOf(ctx, i.leftRow)
 			if err != nil {
 				return nil, err
 			}
@@ -485,12 +485,12 @@ func (i *fullJoinIter) Next(ctx *sql.Context) (sql.Row, error) {
 		if !sql.IsTrue(matches) {
 			continue
 		}
-		rkey, err := sql.HashOf(rightRow)
+		rkey, err := sql.HashOf(ctx, rightRow)
 		if err != nil {
 			return nil, err
 		}
 		i.seenRight[rkey] = struct{}{}
-		lKey, err := sql.HashOf(i.leftRow)
+		lKey, err := sql.HashOf(ctx, i.leftRow)
 		if err != nil {
 			return nil, err
 		}
@@ -517,7 +517,7 @@ func (i *fullJoinIter) Next(ctx *sql.Context) (sql.Row, error) {
 			return nil, io.EOF
 		}
 
-		key, err := sql.HashOf(rightRow)
+		key, err := sql.HashOf(ctx, rightRow)
 		if err != nil {
 			return nil, err
 		}
diff --git a/sql/rowexec/other_iters.go b/sql/rowexec/other_iters.go
@@ -334,7 +334,7 @@ func (ci *concatIter) Next(ctx *sql.Context) (sql.Row, error) {
 		if err != nil {
 			return nil, err
 		}
-		hash, err := sql.HashOf(res)
+		hash, err := sql.HashOf(ctx, res)
 		if err != nil {
 			return nil, err
 		}
diff --git a/sql/rowexec/rel_iters.go b/sql/rowexec/rel_iters.go
@@ -446,7 +446,7 @@ func (r *recursiveCteIter) Next(ctx *sql.Context) (sql.Row, error) {
 
 		var key uint64
 		if r.deduplicate {
-			key, _ = sql.HashOf(row)
+			key, _ = sql.HashOf(ctx, row)
 			if k, _ := r.cache.Get(key); k != nil {
 				// skip duplicate
 				continue
diff --git a/sql/rowexec/update.go b/sql/rowexec/update.go
@@ -248,7 +248,7 @@ func (u *updateJoinIter) Next(ctx *sql.Context) (sql.Row, error) {
 
 			// Determine whether this row in the table has already been updated
 			cache := u.getOrCreateCache(ctx, tableName)
-			hash, err := sql.HashOf(oldTableRow)
+			hash, err := sql.HashOf(ctx, oldTableRow)
 			if err != nil {
 				return nil, err
 			}

Original file line number	Diff line number	Diff line change
`@@ -3046,7 +3046,7 @@ func TestRenameColumn(t *testing.T, harness Harness) {`
`3046`	`3046`	`TestQueryWithContext(t, ctx, e, harness, "ALTER TABLE mydb.tabletest RENAME COLUMN s TO i1", []sql.Row{{types.NewOkResult(0)}}, nil, nil, nil)`
`3047`	`3047`	`TestQueryWithContext(t, ctx, e, harness, "SHOW FULL COLUMNS FROM mydb.tabletest", []sql.Row{`
`3048`	`3048`	`{"i", "int", nil, "NO", "PRI", nil, "", "", ""},`
`3049`		`- {"i1", "varchar(20)", "utf8mb4_0900_bin", "NO", "", nil, "", "", ""},`
	`3049`	`+ {"i1", "text", "utf8mb4_0900_bin", "NO", "", nil, "", "", ""},`
`3050`	`3050`	`}, nil, nil, nil)`
`3051`	`3051`	`})`
`3052`	`3052`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`exec`
`2`	`2`	`create table tabletest (`
`3`	`3`	`i int primary key,`
`4`		`- s varchar(20) not null`
	`4`	`+ s text not null`
`5`	`5`	`)`
`6`	`6`	`----`
`7`	`7`
Original file line number	Diff line number	Diff line change
`@@ -1971,7 +1971,7 @@ func (t Table) DropCheck(ctx sql.Context, chName string) error {`
`1971`	`1971`	`return fmt.Errorf("check '%s' was not found on the table", chName)`
`1972`	`1972`	`}`
`1973`	`1973`
`1974`		`-func (t Table) createIndex(data TableData, name string, columns []sql.IndexColumn, constraint sql.IndexConstraint, comment string) (sql.Index, error) {`
	`1974`	`+func (t Table) createIndex(ctx sql.Context, data *TableData, name string, columns []sql.IndexColumn, constraint sql.IndexConstraint, comment string) (sql.Index, error) {`
`1975`	`1975`	`if name == "" {`
`1976`	`1976`	`for _, column := range columns {`
`1977`	`1977`	`name += column.Name + "_"`
`@@ -2005,7 +2005,7 @@ func (t Table) createIndex(data TableData, name string, columns []sql.IndexCol`
`2005`	`2005`	`}`
`2006`	`2006`
`2007`	`2007`	`if constraint == sql.IndexConstraint_Unique {`
`2008`		`- err := data.errIfDuplicateEntryExist(colNames, name)`
	`2008`	`+ err := data.errIfDuplicateEntryExist(ctx, colNames, name)`
`2009`	`2009`	`if err != nil {`
`2010`	`2010`	`return nil, err`
`2011`	`2011`	`}`
`@@ -2041,7 +2041,7 @@ func (t Table) CreateIndex(ctx sql.Context, idx sql.IndexDef) error {`
`2041`	`2041`	`data.indexes = make(map[string]sql.Index)`
`2042`	`2042`	`}`
`2043`	`2043`
`2044`		`- index, err := t.createIndex(data, idx.Name, idx.Columns, idx.Constraint, idx.Comment)`
	`2044`	`+ index, err := t.createIndex(ctx, data, idx.Name, idx.Columns, idx.Constraint, idx.Comment)`
`2045`	`2045`	`if err != nil {`
`2046`	`2046`	`return err`
`2047`	`2047`	`}`
`@@ -2107,7 +2107,7 @@ func (t Table) CreateFulltextIndex(ctx sql.Context, indexDef sql.IndexDef, key`
`2107`	`2107`	`data.indexes = make(map[string]sql.Index)`
`2108`	`2108`	`}`
`2109`	`2109`
`2110`		`- index, err := t.createIndex(data, indexDef.Name, indexDef.Columns, indexDef.Constraint, indexDef.Comment)`
	`2110`	`+ index, err := t.createIndex(ctx, data, indexDef.Name, indexDef.Columns, indexDef.Constraint, indexDef.Comment)`
`2111`	`2111`	`if err != nil {`
`2112`	`2112`	`return err`
`2113`	`2113`	`}`
`@@ -2138,7 +2138,7 @@ func (t Table) CreateVectorIndex(ctx sql.Context, idx sql.IndexDef, distanceTy`
`2138`	`2138`	`data.indexes = make(map[string]sql.Index)`
`2139`	`2139`	`}`
`2140`	`2140`
`2141`		`- index, err := t.createIndex(data, idx.Name, idx.Columns, idx.Constraint, idx.Comment)`
	`2141`	`+ index, err := t.createIndex(ctx, data, idx.Name, idx.Columns, idx.Constraint, idx.Comment)`
`2142`	`2142`	`if err != nil {`
`2143`	`2143`	`return err`
`2144`	`2144`	`}`
Original file line number	Diff line number	Diff line change
`@@ -571,7 +571,7 @@ func (di distinctIter) Next(ctx sql.Context) (sql.Row, error) {`
`571`	`571`	`return nil, err`
`572`	`572`	`}`
`573`	`573`
`574`		`- hash, err := sql.HashOf(row)`
	`574`	`+ hash, err := sql.HashOf(ctx, row)`
`575`	`575`	`if err != nil {`
`576`	`576`	`return nil, err`
`577`	`577`	`}`
`@@ -647,7 +647,7 @@ func (ii IntersectIter) Next(ctx sql.Context) (sql.Row, error) {`
`647`	`647`	`return nil, err`
`648`	`648`	`}`
`649`	`649`
`650`		`- hash, herr := sql.HashOf(res)`
	`650`	`+ hash, herr := sql.HashOf(ctx, res)`
`651`	`651`	`if herr != nil {`
`652`	`652`	`return nil, herr`
`653`	`653`	`}`
`@@ -669,7 +669,7 @@ func (ii IntersectIter) Next(ctx sql.Context) (sql.Row, error) {`
`669`	`669`	`return nil, err`
`670`	`670`	`}`
`671`	`671`
`672`		`- hash, herr := sql.HashOf(res)`
	`672`	`+ hash, herr := sql.HashOf(ctx, res)`
`673`	`673`	`if herr != nil {`
`674`	`674`	`return nil, herr`
`675`	`675`	`}`
`@@ -714,7 +714,7 @@ func (ei ExceptIter) Next(ctx sql.Context) (sql.Row, error) {`
`714`	`714`	`return nil, err`
`715`	`715`	`}`
`716`	`716`
`717`		`- hash, herr := sql.HashOf(res)`
	`717`	`+ hash, herr := sql.HashOf(ctx, res)`
`718`	`718`	`if herr != nil {`
`719`	`719`	`return nil, herr`
`720`	`720`	`}`
`@@ -736,7 +736,7 @@ func (ei ExceptIter) Next(ctx sql.Context) (sql.Row, error) {`
`736`	`736`	`return nil, err`
`737`	`737`	`}`
`738`	`738`
`739`		`- hash, herr := sql.HashOf(res)`
	`739`	`+ hash, herr := sql.HashOf(ctx, res)`
`740`	`740`	`if herr != nil {`
`741`	`741`	`return nil, herr`
`742`	`742`	`}`