Add a contains operator to the VAI filter language (#1002)

* Implement and test code-gen for a CONTAINS operator.

* Lex and parse a CONTAINS operator.

* Add the 'hasBarredValue' custom sql function.

* Add integration tests for the CONTAINS operator.

* Add integration tests that verify 'contains' doesn't pick substrings

* Add the NOTCONTAINS operator.

Also:
- Simplify code-gen for `metadata.labels.... CONTAINS ...`
- Because labels can't have '|' characters, CONTAINS reduces to ==

* Add integration tests for the CONTAINS op.

Author: ericpromislow

pkg/sqlcache/db/client.go

@@ -18,6 +18,7 @@ import (
 	"os"
 	"reflect"
 	"regexp"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -544,6 +545,7 @@ func (c *client) NewConnection(useTempDir bool) (string, error) {
 		return dbPath, err
 	}
 	sqlite.RegisterDeterministicScalarFunction("extractBarredValue", 2, extractBarredValue)
+	sqlite.RegisterDeterministicScalarFunction("hasBarredValue", 2, hasBarredValue)
 	sqlite.RegisterDeterministicScalarFunction("inet_aton", 1, inetAtoN)
 	sqlite.RegisterDeterministicScalarFunction("memoryInBytes", 1, memoryInBytes)
 	c.conn = &connection{sqlDB}
@@ -573,7 +575,7 @@ func extractBarredValue(ctx *sqlite.FunctionContext, args []driver.Value) (drive
 		return nil, fmt.Errorf("unsupported type for arg2: expected an int, got: %T", args[0])
 	}
 	if err != nil {
-		return nil, fmt.Errorf("problem with arg2: %w", err)
+		return nil, fmt.Errorf("extractBarredValue: problem with arg2: %w", err)
 	}
 	parts := strings.Split(arg1, "|")
 	if arg2 >= len(parts) || arg2 < 0 {
@@ -582,6 +584,29 @@ func extractBarredValue(ctx *sqlite.FunctionContext, args []driver.Value) (drive
 	return parts[arg2], nil
 }
 
+func hasBarredValue(ctx *sqlite.FunctionContext, args []driver.Value) (driver.Value, error) {
+	var arg1 string
+	var arg2 string
+	switch argTyped := args[0].(type) {
+	case string:
+		arg1 = argTyped
+	case []byte:
+		arg1 = string(argTyped)
+	default:
+		return nil, fmt.Errorf("hasBarredValue: unsupported type for arg1: expected a string, got :%T", args[0])
+	}
+	switch argTyped := args[1].(type) {
+	case string:
+		arg2 = argTyped
+	case []byte:
+		arg2 = string(argTyped)
+	default:
+		return nil, fmt.Errorf("hasBarredValue: unsupported type for arg2: expected a string, got: %T", args[0])
+	}
+	parts := strings.Split(arg1, "|")
+	return slices.Contains(parts, arg2), nil
+}
+
 func inetAtoN(ctx *sqlite.FunctionContext, args []driver.Value) (driver.Value, error) {
 	var arg1 string
 	switch argTyped := args[0].(type) {

pkg/sqlcache/informer/sqlgenerator.go

@@ -887,6 +887,8 @@ SELECT key, value FROM "%s_labels"
 // KEY ! []  # ,!KEY, => assert KEY doesn't exist
 // KEY in VALUES
 // KEY notin VALUES
+// KEY contains VALUES
+// KEY notcontains VALUES
 
 func (l *ListOptionIndexer) getFieldFilter(filter sqltypes.Filter, prefix string) (string, []any, error) {
 	opString := ""
@@ -946,6 +948,24 @@ func (l *ListOptionIndexer) getFieldFilter(filter sqltypes.Filter, prefix string
 			matches[i] = match
 		}
 		return clause, matches, nil
+
+	case sqltypes.Contains:
+		if len(filter.Matches) != 1 {
+			return "", nil, fmt.Errorf("array checking works on exactly one field, %d were specified", len(filter.Matches))
+		}
+		clause := fmt.Sprintf("hasBarredValue(%s, ?)", fieldEntry)
+		matches := make([]any, 1)
+		matches[0] = filter.Matches[0]
+		return clause, matches, nil
+
+	case sqltypes.NotContains:
+		if len(filter.Matches) != 1 {
+			return "", nil, fmt.Errorf("array checking works on exactly one field, %d were specified", len(filter.Matches))
+		}
+		clause := fmt.Sprintf("NOT hasBarredValue(%s, ?)", fieldEntry)
+		matches := make([]any, 1)
+		matches[0] = filter.Matches[0]
+		return clause, matches, nil
 	}
 
 	return "", nil, fmt.Errorf("unrecognized operator: %s", opString)
@@ -1052,6 +1072,22 @@ func (l *ListOptionIndexer) getLabelFilter(index int, filter sqltypes.Filter, ma
 			matches = append(matches, match)
 		}
 		return clause, matches, nil
+
+	case sqltypes.Contains:
+		if len(filter.Matches) != 1 {
+			return "", nil, fmt.Errorf("array checking works on exactly one field, %d were specified", len(filter.Matches))
+		}
+		// Labels can't have | characters so they're implemented like '='
+		filter.Op = sqltypes.Eq
+		return l.getLabelFilter(index, filter, mainFieldPrefix, isSummaryFilter, dbName)
+
+	case sqltypes.NotContains:
+		if len(filter.Matches) != 1 {
+			return "", nil, fmt.Errorf("array checking works on exactly one field, %d were specified", len(filter.Matches))
+		}
+		// Labels can't have | characters so they're implemented like '='
+		filter.Op = sqltypes.NotEq
+		return l.getLabelFilter(index, filter, mainFieldPrefix, isSummaryFilter, dbName)
 	}
 	return "", nil, fmt.Errorf("unrecognized operator: %s", opString)
 }

pkg/sqlcache/informer/sqlgenerator_test.go

@@ -2566,6 +2566,268 @@ SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o
 	}
 }
 
+func TestConstructQueryWithContainsOp(t *testing.T) {
+	type testCase struct {
+		description           string
+		listOptions           sqltypes.ListOptions
+		partitions            []partition.Partition
+		ns                    string
+		expectedCountStmt     string
+		expectedCountStmtArgs []any
+		expectedStmt          string
+		expectedStmtArgs      []any
+		expectedErr           string
+	}
+
+	var tests []testCase
+	tests = append(tests, testCase{
+		description: "TestConstructQueryWithContainsOp: handles CONTAIN statements on indexed arrays",
+		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{
+			{
+				[]sqltypes.Filter{
+					{
+						Matches: []string{"needle01"},
+						Field:   []string{"metadata", "fields"},
+						Op:      sqltypes.Contains,
+					},
+				},
+			},
+		},
+		},
+		partitions: []partition.Partition{},
+		ns:         "",
+		expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o
+  JOIN "something_fields" f ON o.key = f.key
+  WHERE
+    (hasBarredValue(f."metadata.fields", ?)) AND
+    (FALSE)
+  ORDER BY f."metadata.name" ASC`,
+		expectedStmtArgs: []any{"needle01"},
+	})
+	tests = append(tests, testCase{
+		description: "TestConstructQueryWithContainsOp: handles CONTAIN statements on single strings",
+		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{
+			{
+				[]sqltypes.Filter{
+					{
+						Matches: []string{"needle02"},
+						Field:   []string{"metadata", "queryField1"},
+						Op:      sqltypes.Contains,
+					},
+				},
+			},
+		},
+		},
+		partitions: []partition.Partition{},
+		ns:         "",
+		expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o
+  JOIN "something_fields" f ON o.key = f.key
+  WHERE
+    (hasBarredValue(f."metadata.queryField1", ?)) AND
+    (FALSE)
+  ORDER BY f."metadata.name" ASC`,
+		expectedStmtArgs: []any{"needle02"},
+	})
+	tests = append(tests, testCase{
+		description: "TestConstructQueryWithContainsOp: error CONTAIN statements on too many target strings",
+		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{
+			{
+				[]sqltypes.Filter{
+					{
+						Matches: []string{"too", "many", "targets"},
+						Field:   []string{"metadata", "queryField1"},
+						Op:      sqltypes.Contains,
+					},
+				},
+			},
+		},
+		},
+		ns:          "",
+		expectedErr: "array checking works on exactly one field, 3 were specified",
+	})
+	tests = append(tests, testCase{
+		description: "TestConstructQueryWithContainsOp: error CONTAIN statements on unrecognized field",
+		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{
+			{
+				[]sqltypes.Filter{
+					{
+						Matches: []string{"this"},
+						Field:   []string{"bills", "farm"},
+						Op:      sqltypes.Contains,
+					},
+				},
+			},
+		},
+		},
+		ns:          "",
+		expectedErr: "column is invalid [bills.farm]: supplied column is invalid",
+	})
+	tests = append(tests, testCase{
+		description: "TestConstructQueryWithContainsOp: error CONTAIN statements on no target string",
+		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{
+			{
+				[]sqltypes.Filter{
+					{
+						Field: []string{"metadata", "queryField1"},
+						Op:    sqltypes.Contains,
+					},
+				},
+			},
+		},
+		},
+		ns:          "",
+		expectedErr: "array checking works on exactly one field, 0 were specified",
+	})
+	tests = append(tests, testCase{
+		description: "TestConstructQueryWithContainsOp: handles NOTCONTAINS statements on indexed arrays",
+		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{
+			{
+				[]sqltypes.Filter{
+					{
+						Matches: []string{"needle01"},
+						Field:   []string{"metadata", "fields"},
+						Op:      sqltypes.NotContains,
+					},
+				},
+			},
+		},
+		},
+		partitions: []partition.Partition{},
+		ns:         "",
+		expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o
+  JOIN "something_fields" f ON o.key = f.key
+  WHERE
+    (NOT hasBarredValue(f."metadata.fields", ?)) AND
+    (FALSE)
+  ORDER BY f."metadata.name" ASC`,
+		expectedStmtArgs: []any{"needle01"},
+	})
+	tests = append(tests, testCase{
+		description: "TestConstructQueryWithContainsOp: handles NOTCONTAINS statements on single strings",
+		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{
+			{
+				[]sqltypes.Filter{
+					{
+						Matches: []string{"needle02"},
+						Field:   []string{"metadata", "queryField1"},
+						Op:      sqltypes.NotContains,
+					},
+				},
+			},
+		},
+		},
+		partitions: []partition.Partition{},
+		ns:         "",
+		expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o
+  JOIN "something_fields" f ON o.key = f.key
+  WHERE
+    (NOT hasBarredValue(f."metadata.queryField1", ?)) AND
+    (FALSE)
+  ORDER BY f."metadata.name" ASC`,
+		expectedStmtArgs: []any{"needle02"},
+	})
+	tests = append(tests, testCase{
+		description: "TestConstructQueryWithContainsOp: error NOTCONTAINS statements on too many target strings",
+		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{
+			{
+				[]sqltypes.Filter{
+					{
+						Matches: []string{"too", "many", "targets"},
+						Field:   []string{"metadata", "queryField1"},
+						Op:      sqltypes.NotContains,
+					},
+				},
+			},
+		},
+		},
+		ns:          "",
+		expectedErr: "array checking works on exactly one field, 3 were specified",
+	})
+	tests = append(tests, testCase{
+		description: "TestConstructQueryWithContainsOp: error NOTCONTAINS statements on unrecognized field",
+		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{
+			{
+				[]sqltypes.Filter{
+					{
+						Matches: []string{"this"},
+						Field:   []string{"bills", "farm"},
+						Op:      sqltypes.NotContains,
+					},
+				},
+			},
+		},
+		},
+		ns:          "",
+		expectedErr: "column is invalid [bills.farm]: supplied column is invalid",
+	})
+	tests = append(tests, testCase{
+		description: "TestConstructQueryWithContainsOp: error NOTCONTAINS statements on no target string",
+		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{
+			{
+				[]sqltypes.Filter{
+					{
+						Field: []string{"metadata", "queryField1"},
+						Op:    sqltypes.NotContains,
+					},
+				},
+			},
+		},
+		},
+		ns:          "",
+		expectedErr: "array checking works on exactly one field, 0 were specified",
+	})
+	tests = append(tests, testCase{
+		description: "TestConstructQueryWithContainsOp: handles label: NOTCONTAINS statements on single strings",
+		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{
+			{
+				[]sqltypes.Filter{
+					{
+						Matches: []string{"needle03"},
+						Field:   []string{"metadata", "labels", "sewingSupplies"},
+						Op:      sqltypes.NotContains,
+					},
+				},
+			},
+		},
+		},
+		partitions: []partition.Partition{},
+		ns:         "",
+		expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o
+  JOIN "something_fields" f ON o.key = f.key
+  LEFT OUTER JOIN "something_labels" lt1 ON f.key = lt1.key
+  WHERE
+    ((o.key NOT IN (SELECT f1.key FROM "something_fields" f1
+		LEFT OUTER JOIN "something_labels" lt1i1 ON f1.key = lt1i1.key
+		WHERE lt1i1.label = ?)) OR (lt1.label = ? AND lt1.value != ?)) AND
+    (FALSE)
+  ORDER BY f."metadata.name" ASC`,
+		expectedStmtArgs: []any{"sewingSupplies", "sewingSupplies", "needle03"},
+	})
+	t.Parallel()
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			store := NewMockStore(gomock.NewController(t))
+			i := &Indexer{
+				Store: store,
+			}
+			lii := &ListOptionIndexer{
+				Indexer:       i,
+				indexedFields: toIndexedFieldsFromColumnNames("metadata.queryField1", "status.queryField2", "metadata.fields"),
+			}
+			queryInfo, err := lii.constructQuery(&test.listOptions, test.partitions, test.ns, "something")
+			if test.expectedErr != "" {
+				assert.Equal(t, test.expectedErr, err.Error())
+				return
+			}
+			require.Nil(t, err)
+			assert.Equal(t, test.expectedStmt, queryInfo.query)
+			assert.Equal(t, test.expectedStmtArgs, queryInfo.params)
+			assert.Equal(t, test.expectedCountStmt, queryInfo.countQuery)
+			assert.Equal(t, test.expectedCountStmtArgs, queryInfo.countParams)
+		})
+	}
+}
+
 func TestConstructSummaryQueryForField(t *testing.T) {
 	type testCase struct {
 		description      string