opt: fix optional filter removal for lookup join

Fixes #159660

Previously, we don't reset the `optionalMultiValFilterSuffixLen` when we
find a necessary multi-val filter for a column. This would cause an incorrect
removal of an unoptional filter.

For example,
- Index: (x, y ,z)
- ON filters: x = 1, z IN (1, 2)
- Optional filters: y IN (3, 4)

Since the filters are processed with the order of columns in the index,
the filters will be processed in the order of `x = 1`, `y IN (3, 4)`,
`z IN (1, 2)`.

When the processing reaches `y IN (3, 4)`, since it is an optional filter,
`optionalMultiValFilterSuffixLen = 1`. But `z IN (1, 2)` is actually a
necessary filter, with `optionalMultiValFilterSuffixLen` not being reset,
it will be eventually wrongly removed.

Release note (bug fix): Fixed a bug causing a query predicate to be ignored
when the predicate is on a column following one or more ENUM columns in an
index, the predicate constrains the column to multiple values, and a lookup
join to the index is chosen for the query plan. This bug was introduced in
24.3.0 and present in all versions since.

Author: ZhouXing19

pkg/sql/opt/lookupjoin/constraint_builder.go

@@ -348,6 +348,7 @@ func (b *ConstraintBuilder) Build(
 			if isOptional := allIdx >= len(onFilters); isOptional {
 				optionalMultiValFilterSuffixLen++
 			} else {
+				optionalMultiValFilterSuffixLen = 0
 				// There's no need to track optional filters for reducing the
 				// remaining filters because they are not present in the ON
 				// filters to begin with.

pkg/sql/opt/lookupjoin/testdata/key_cols

@@ -191,6 +191,13 @@ optional: z IN (3, 4) AND y IN (10, 20)
 key cols:
   x = a
 
+lookup-constraints left=(a int, b int, c int) right=(x int, y int, z int) index=(x, y, z)
+x = a AND z IN (3, 4)
+optional: y IN (10, 20)
+----
+lookup expression:
+  ((y IN (10, 20)) AND (z IN (3, 4))) AND (a = x)
+
 lookup-constraints left=(a int, b int) right=(x int, y int, z int) index=(x, y, z)
 x = a AND y = b
 optional: z > 10

pkg/sql/opt/lookupjoin/testdata/lookup_expr

@@ -489,3 +489,10 @@ optional: y IN (10, 20, 30)
 ----
 lookup expression:
   ((y IN (10, 20, 30)) AND (z > b)) AND (a = x)
+
+lookup-constraints left=(a int, b int, c int) right=(x int, y int, z int) index=(x, y, z)
+x = a AND z IN (100, 200, 300)
+optional: y IN (10, 20, 30)
+----
+lookup expression:
+  ((y IN (10, 20, 30)) AND (z IN (100, 200, 300))) AND (a = x)

pkg/sql/opt/xform/testdata/rules/join

@@ -3123,6 +3123,38 @@ left-join (lookup lookup_expr [as=t])
  │         └── column1:1 = w:7 [outer=(1,7), constraints=(/1: (/NULL - ]; /7: (/NULL - ]), fd=(1)==(7), (7)==(1)]
  └── filters (true)
 
+# The filters in the lookup expression should contains all columns of the
+# idx_vrw index. Column r, though not in the ON exprs, should still be
+# in the filters, because for the column w that comes behind it in
+# idx_vrw, there is a necessary filter `w IN (10, 20)`.
+opt expect=GenerateLookupJoinsWithFilter
+SELECT * FROM (VALUES (1), (2), (3)) AS q(v) LEFT LOOKUP JOIN lookup_expr t
+ON q.v = t.v WHERE w IN (10, 20)
+----
+inner-join (lookup lookup_expr [as=t])
+ ├── columns: v:1!null r:2!null k:3!null u:4 v:5!null w:6!null x:7 y:8!null z:9!null
+ ├── key columns: [2 3] = [2 3]
+ ├── lookup columns are key
+ ├── fd: ()-->(9), (2,3)-->(4-8), (1)==(5), (5)==(1)
+ ├── inner-join (lookup lookup_expr@idx_vrw [as=t])
+ │    ├── columns: column1:1!null r:2!null k:3!null v:5!null w:6!null
+ │    ├── flags: force lookup join (into right side)
+ │    ├── lookup expression
+ │    │    └── filters
+ │    │         ├── r:2 IN ('east', 'west') [outer=(2), constraints=(/2: [/'east' - /'east'] [/'west' - /'west']; tight)]
+ │    │         ├── w:6 IN (10, 20) [outer=(6), constraints=(/6: [/10 - /10] [/20 - /20]; tight)]
+ │    │         └── column1:1 = v:5 [outer=(1,5), constraints=(/1: (/NULL - ]; /5: (/NULL - ]), fd=(1)==(5), (5)==(1)]
+ │    ├── fd: (2,3)-->(5,6), (1)==(5), (5)==(1)
+ │    ├── values
+ │    │    ├── columns: column1:1!null
+ │    │    ├── cardinality: [3 - 3]
+ │    │    ├── (1,)
+ │    │    ├── (2,)
+ │    │    └── (3,)
+ │    └── filters (true)
+ └── filters (true)
+
+
 # Ensure that we constrain all columns in idx_vrw, not just v.
 opt expect=GenerateLookupJoins
 SELECT * FROM (VALUES (1, 10), (2, 20), (3, NULL)) AS q(v, w) LEFT JOIN lookup_expr t