Merge #147045

147045: sql: add support for like with collated columns r=paulniziolek a=paulniziolek

#### sql: add support for like with collated columns 
Fixes: #20666 
Epic: CRDB-5918 
Release note (sql change): Deterministic collations are now supported with
LIKE. A deterministic collation considers strings to be equal only if they
consist of the same byte sequence.


Co-authored-by: Paul Niziolek <[email protected]>

Author: craig[bot]

docs/generated/sql/operators.md

@@ -513,6 +513,7 @@
 <table><thead>
 <tr><td><code>LIKE</code></td><td>Return</td></tr>
 </thead><tbody>
+<tr><td><a href="collate.html">collatedstring</a> <code>LIKE</code> <a href="collate.html">collatedstring</a></td><td><a href="bool.html">bool</a></td></tr>
 <tr><td><a href="string.html">string</a> <code>LIKE</code> <a href="string.html">string</a></td><td><a href="bool.html">bool</a></td></tr>
 </tbody></table>
 <table><thead>

pkg/BUILD.bazel

@@ -733,6 +733,7 @@ ALL_TESTS = [
     "//pkg/util/cidr:cidr_test",
     "//pkg/util/circuit:circuit_test",
     "//pkg/util/cloudinfo:cloudinfo_test",
+    "//pkg/util/collatedstring:collatedstring_test",
     "//pkg/util/container/heap:heap_test",
     "//pkg/util/container/list:list_test",
     "//pkg/util/container/ring:ring_test",
@@ -2577,6 +2578,7 @@ GO_TARGETS = [
     "//pkg/util/cloudinfo:cloudinfo",
     "//pkg/util/cloudinfo:cloudinfo_test",
     "//pkg/util/collatedstring:collatedstring",
+    "//pkg/util/collatedstring:collatedstring_test",
     "//pkg/util/container/heap:heap",
     "//pkg/util/container/heap:heap_test",
     "//pkg/util/container/list:list",

pkg/sql/logictest/testdata/logic_test/collatedstring

@@ -762,3 +762,9 @@ SELECT b COLLATE "en_US_u_ks_level2" FROM str_arr ORDER BY 1
 NULL
 d
 h
+
+query error unsupported comparison operator: <collatedstring{en_US}> LIKE <collatedstring{de_DE}>
+SELECT 'TEST' COLLATE "en_US" LIKE 'TEST' COLLATE "de_DE"
+
+query error nondeterministic collations are not supported for LIKE
+SELECT 'TEST' COLLATE "en_US-u-ks-level1" LIKE 'TEST' COLLATE "en_US-u-ks-level1"

pkg/sql/pgwire/pgwire_test.go

@@ -478,7 +478,7 @@ func TestPGPreparedQuery(t *testing.T) {
 			baseTest.SetArgs(2, 3).Results(2),
 			baseTest.SetArgs(true, 0).Error(`error in argument for \$1: could not parse "true" as type int: strconv.ParseInt: parsing "true": invalid syntax`),
 		}},
-		{"SELECT $1[2] LIKE 'b'", []preparedQueryTest{
+		{"SELECT ($1::TEXT[])[2] LIKE 'b'", []preparedQueryTest{
 			baseTest.SetArgs(pq.Array([]string{"a", "b", "c"})).Results(true),
 			baseTest.SetArgs(pq.Array([]gosql.NullString{{String: "a", Valid: true}, {Valid: false}, {String: "c", Valid: true}})).Results(gosql.NullBool{Valid: false}),
 		}},

pkg/sql/rowexec/sampler_test.go

@@ -308,7 +308,7 @@ func TestSamplerSketch(t *testing.T) {
 			},
 			cardinalities: []int{3, 2, 3},
 			numNulls:      []int{1, 1, 1},
-			size:          []int{352, 352, 704},
+			size:          []int{384, 384, 768},
 			keySize:       []int{89, 89, 178},
 			valueSize:     []int{21, 21, 42},
 		},

pkg/sql/sem/eval/match.go

@@ -81,7 +81,17 @@ func matchLike(ctx *Context, left, right tree.Datum, caseInsensitive bool) (tree
 	if left == tree.DNull || right == tree.DNull {
 		return tree.DNull, nil
 	}
-	s, pattern := string(tree.MustBeDString(left)), string(tree.MustBeDString(right))
+	var s, pattern string
+	var err error
+	s, err = matchStringFromDatum(left)
+	if err != nil {
+		return tree.DBoolFalse, err
+	}
+	pattern, err = matchStringFromDatum(right)
+	if err != nil {
+		return tree.DBoolFalse, err
+	}
+
 	if len(s) == 0 {
 		// An empty string only matches with an empty pattern or a pattern
 		// consisting only of '%'. To match PostgreSQL's behavior, we have a
@@ -113,6 +123,18 @@ func matchLike(ctx *Context, left, right tree.Datum, caseInsensitive bool) (tree
 	return tree.MakeDBool(tree.DBool(matches)), err
 }
 
+func matchStringFromDatum(datum tree.Datum) (string, error) {
+	switch d := datum.(type) {
+	case *tree.DCollatedString:
+		if !d.Deterministic {
+			return "", pgerror.New(pgcode.FeatureNotSupported, "nondeterministic collations are not supported for LIKE")
+		}
+		return d.Contents, nil
+	default:
+		return string(tree.MustBeDString(d)), nil
+	}
+}
+
 func matchRegexpWithKey(ctx *Context, str tree.Datum, key tree.RegexpCacheKey) (tree.Datum, error) {
 	re, err := ctx.ReCache.GetRegexp(key)
 	if err != nil {

pkg/sql/sem/eval/testdata/eval/like

@@ -1350,3 +1350,43 @@ eval
 'TEST' NOT ILIKE 'TE_'
 ----
 true
+
+eval
+'TEST' COLLATE "en_US" LIKE 'TEST' COLLATE "en_US"
+----
+true
+
+eval
+'TEST' COLLATE "en_US" LIKE 'TEST'
+----
+true
+
+eval
+'TEST' LIKE 'TEST' COLLATE "en_US"
+----
+true
+
+eval
+'TEST' COLLATE "en_US" LIKE 'TESTER' COLLATE "en_US"
+----
+false
+
+eval
+'TEST' COLLATE "en_US" LIKE '_ES%'
+----
+true
+
+eval
+'TEST' COLLATE "en_US" LIKE 'test' COLLATE "de_DE"
+----
+unsupported comparison operator: <collatedstring{en_US}> LIKE <collatedstring{de_DE}>
+
+eval
+'TEST' COLLATE "en_US_u-ks-level1" LIKE 'test' COLLATE "en_US-u-ks-level1"
+----
+nondeterministic collations are not supported for LIKE
+
+eval
+'TEST' COLLATE "en_US-ks-level1" LIKE 'test' COLLATE "en_US-ks-level1"
+----
+invalid locale en_US-ks-level1: language: tag is not well-formed

pkg/sql/sem/tree/datum.go

@@ -32,6 +32,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/types"
 	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/bitarray"
+	"github.com/cockroachdb/cockroach/pkg/util/collatedstring"
 	"github.com/cockroachdb/cockroach/pkg/util/duration"
 	"github.com/cockroachdb/cockroach/pkg/util/encoding"
 	"github.com/cockroachdb/cockroach/pkg/util/ipaddr"
@@ -1322,7 +1323,8 @@ type DCollatedString struct {
 	Contents string
 	Locale   string
 	// Key is the collation key.
-	Key []byte
+	Key           []byte
+	Deterministic bool
 }
 
 // CollationEnvironment stores the state needed by NewDCollatedString to
@@ -1336,7 +1338,8 @@ type collationEnvironmentCacheEntry struct {
 	// locale is interned.
 	locale string
 	// collator is an expensive factory.
-	collator *collate.Collator
+	collator      *collate.Collator
+	deterministic bool
 }
 
 func (env *CollationEnvironment) getCacheEntry(
@@ -1353,7 +1356,7 @@ func (env *CollationEnvironment) getCacheEntry(
 			return collationEnvironmentCacheEntry{}, err
 		}
 
-		entry = collationEnvironmentCacheEntry{locale, collate.New(tag)}
+		entry = collationEnvironmentCacheEntry{locale, collate.New(tag), collatedstring.IsDeterministicCollation(tag)}
 		env.cache[locale] = entry
 	}
 	return entry, nil
@@ -1372,7 +1375,7 @@ func NewDCollatedString(
 		env.buffer = &collate.Buffer{}
 	}
 	key := entry.collator.KeyFromString(env.buffer, contents)
-	d := DCollatedString{contents, entry.locale, make([]byte, len(key))}
+	d := DCollatedString{contents, entry.locale, make([]byte, len(key)), entry.deterministic}
 	copy(d.Key, key)
 	env.buffer.Reset()
 	return &d, nil
@@ -1445,7 +1448,7 @@ func (d *DCollatedString) IsMin(ctx context.Context, cmpCtx CompareContext) bool
 
 // Min implements the Datum interface.
 func (d *DCollatedString) Min(ctx context.Context, cmpCtx CompareContext) (Datum, bool) {
-	return &DCollatedString{"", d.Locale, nil}, true
+	return &DCollatedString{"", d.Locale, nil, false}, true
 }
 
 // Max implements the Datum interface.
@@ -6204,7 +6207,7 @@ var baseDatumTypeSizes = map[types.Family]struct {
 	types.FloatFamily:          {unsafe.Sizeof(DFloat(0.0)), fixedSize},
 	types.DecimalFamily:        {unsafe.Sizeof(DDecimal{}), variableSize},
 	types.StringFamily:         {unsafe.Sizeof(DString("")), variableSize},
-	types.CollatedStringFamily: {unsafe.Sizeof(DCollatedString{"", "", nil}), variableSize},
+	types.CollatedStringFamily: {unsafe.Sizeof(DCollatedString{"", "", nil, false}), variableSize},
 	types.BytesFamily:          {unsafe.Sizeof(DBytes("")), variableSize},
 	types.EncodedKeyFamily:     {unsafe.Sizeof(DBytes("")), variableSize},
 	types.DateFamily:           {unsafe.Sizeof(DDate{}), fixedSize},

pkg/sql/sem/tree/eval.go

@@ -1926,12 +1926,9 @@ var CmpOps = cmpOpFixups(map[treecmp.ComparisonOperatorSymbol]*CmpOpOverloads{
 	}},
 
 	treecmp.Like: {overloads: []*CmpOp{
-		{
-			LeftType:   types.String,
-			RightType:  types.String,
-			EvalOp:     &MatchLikeOp{CaseInsensitive: false},
-			Volatility: volatility.Leakproof,
-		},
+		// TODO(mgartner): This overload should be immutable, not leakproof.
+		makeLikeFn(types.String, types.String, volatility.Leakproof),
+		makeLikeFn(types.AnyCollatedString, types.AnyCollatedString, volatility.Immutable),
 	}},
 
 	treecmp.ILike: {overloads: []*CmpOp{
@@ -2148,6 +2145,15 @@ func makeEvalTupleIn(typ *types.T, v volatility.V) *CmpOp {
 	}
 }
 
+func makeLikeFn(a, b *types.T, v volatility.V) *CmpOp {
+	return &CmpOp{
+		LeftType:   a,
+		RightType:  b,
+		EvalOp:     &MatchLikeOp{CaseInsensitive: false},
+		Volatility: v,
+	}
+}
+
 // MultipleResultsError is returned by QueryRow when more than one result is
 // encountered.
 type MultipleResultsError struct {

pkg/util/collatedstring/BUILD.bazel

@@ -1,9 +1,22 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "collatedstring",
     srcs = ["collatedstring.go"],
     importpath = "github.com/cockroachdb/cockroach/pkg/util/collatedstring",
     visibility = ["//visibility:public"],
-    deps = ["@org_golang_x_text//collate"],
+    deps = [
+        "@org_golang_x_text//collate",
+        "@org_golang_x_text//language",
+    ],
+)
+
+go_test(
+    name = "collatedstring_test",
+    srcs = ["collatedstring_test.go"],
+    embed = [":collatedstring"],
+    deps = [
+        "@com_github_stretchr_testify//require",
+        "@org_golang_x_text//language",
+    ],
 )