feat(filter): add CEL list comprehension support for tag filtering

johnnyjoygh · johnnyjoygh · commit cbf46a298836 · 2026-01-17T12:36:39.000+08:00
Add support for CEL exists() comprehension with startsWith, endsWith, and contains predicates to enable powerful tag filtering patterns. Features: - tags.exists(t, t.startsWith("prefix")) - Match tags by prefix - tags.exists(t, t.endsWith("suffix")) - Match tags by suffix - tags.exists(t, t.contains("substring")) - Match tags by substring - Negation: !tags.exists(...) to exclude matching tags - Works with all operators (AND, OR, NOT) and other filters Implementation: - Added ListComprehensionCondition IR type for comprehension expressions - Parser detects exists() macro and extracts predicates - Renderer generates optimized SQL for SQLite, MySQL, PostgreSQL - Proper NULL/empty array handling across all database dialects - Helper functions reduce code duplication Design decisions: - Only exists() supported (all() rejected at parse time with clear error) - Only simple predicates (matches() excluded to avoid regex complexity) - Fail-fast validation with helpful error messages Tests: - Comprehensive test suite covering all predicates and edge cases - Tests for NULL/empty arrays, combined filters, negation - Real-world use case test for Issue #5480 (archive workflow) - All tests pass on SQLite, MySQL, PostgreSQL Closes #5480
diff --git a/plugin/filter/ir.go b/plugin/filter/ir.go
@@ -114,3 +114,46 @@ type FunctionValue struct {
 }
 
 func (*FunctionValue) isValueExpr() {}
+
+// ListComprehensionCondition represents CEL macros like exists(), all(), filter().
+type ListComprehensionCondition struct {
+	Kind      ComprehensionKind
+	Field     string        // The list field to iterate over (e.g., "tags")
+	IterVar   string        // The iteration variable name (e.g., "t")
+	Predicate PredicateExpr // The predicate to evaluate for each element
+}
+
+func (*ListComprehensionCondition) isCondition() {}
+
+// ComprehensionKind enumerates the types of list comprehensions.
+type ComprehensionKind string
+
+const (
+	ComprehensionExists ComprehensionKind = "exists"
+)
+
+// PredicateExpr represents predicates used in comprehensions.
+type PredicateExpr interface {
+	isPredicateExpr()
+}
+
+// StartsWithPredicate represents t.startsWith("prefix").
+type StartsWithPredicate struct {
+	Prefix string
+}
+
+func (*StartsWithPredicate) isPredicateExpr() {}
+
+// EndsWithPredicate represents t.endsWith("suffix").
+type EndsWithPredicate struct {
+	Suffix string
+}
+
+func (*EndsWithPredicate) isPredicateExpr() {}
+
+// ContainsPredicate represents t.contains("substring").
+type ContainsPredicate struct {
+	Substring string
+}
+
+func (*ContainsPredicate) isPredicateExpr() {}
diff --git a/plugin/filter/parser.go b/plugin/filter/parser.go
@@ -36,6 +36,8 @@ func buildCondition(expr *exprv1.Expr, schema Schema) (Condition, error) {
 			return nil, errors.Errorf("identifier %q is not boolean", name)
 		}
 		return &FieldPredicateCondition{Field: name}, nil
+	case *exprv1.Expr_ComprehensionExpr:
+		return buildComprehensionCondition(v.ComprehensionExpr, schema)
 	default:
 		return nil, errors.New("unsupported top-level expression")
 	}
@@ -415,3 +417,170 @@ func evaluateNumeric(expr *exprv1.Expr) (int64, bool, error) {
 func timeNowUnix() int64 {
 	return time.Now().Unix()
 }
+
+// buildComprehensionCondition handles CEL comprehension expressions (exists, all, etc.).
+func buildComprehensionCondition(comp *exprv1.Expr_Comprehension, schema Schema) (Condition, error) {
+	// Determine the comprehension kind by examining the loop initialization and step
+	kind, err := detectComprehensionKind(comp)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the field being iterated over
+	iterRangeIdent := comp.IterRange.GetIdentExpr()
+	if iterRangeIdent == nil {
+		return nil, errors.New("comprehension range must be a field identifier")
+	}
+	fieldName := iterRangeIdent.GetName()
+
+	// Validate the field
+	field, ok := schema.Field(fieldName)
+	if !ok {
+		return nil, errors.Errorf("unknown field %q in comprehension", fieldName)
+	}
+	if field.Kind != FieldKindJSONList {
+		return nil, errors.Errorf("field %q does not support comprehension (must be a list)", fieldName)
+	}
+
+	// Extract the predicate from the loop step
+	predicate, err := extractPredicate(comp, schema)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ListComprehensionCondition{
+		Kind:      kind,
+		Field:     fieldName,
+		IterVar:   comp.IterVar,
+		Predicate: predicate,
+	}, nil
+}
+
+// detectComprehensionKind determines if this is an exists() macro.
+// Only exists() is currently supported.
+func detectComprehensionKind(comp *exprv1.Expr_Comprehension) (ComprehensionKind, error) {
+	// Check the accumulator initialization
+	accuInit := comp.AccuInit.GetConstExpr()
+	if accuInit == nil {
+		return "", errors.New("comprehension accumulator must be initialized with a constant")
+	}
+
+	// exists() starts with false and uses OR (||) in loop step
+	if accuInit.GetBoolValue() == false {
+		if step := comp.LoopStep.GetCallExpr(); step != nil && step.Function == "_||_" {
+			return ComprehensionExists, nil
+		}
+	}
+
+	// all() starts with true and uses AND (&&) - not supported
+	if accuInit.GetBoolValue() == true {
+		if step := comp.LoopStep.GetCallExpr(); step != nil && step.Function == "_&&_" {
+			return "", errors.New("all() comprehension is not supported; use exists() instead")
+		}
+	}
+
+	return "", errors.New("unsupported comprehension type; only exists() is supported")
+}
+
+// extractPredicate extracts the predicate expression from the comprehension loop step.
+func extractPredicate(comp *exprv1.Expr_Comprehension, schema Schema) (PredicateExpr, error) {
+	// The loop step is: @result || predicate(t) for exists
+	//                or: @result && predicate(t) for all
+	step := comp.LoopStep.GetCallExpr()
+	if step == nil {
+		return nil, errors.New("comprehension loop step must be a call expression")
+	}
+
+	if len(step.Args) != 2 {
+		return nil, errors.New("comprehension loop step must have two arguments")
+	}
+
+	// The predicate is the second argument
+	predicateExpr := step.Args[1]
+	predicateCall := predicateExpr.GetCallExpr()
+	if predicateCall == nil {
+		return nil, errors.New("comprehension predicate must be a function call")
+	}
+
+	// Handle different predicate functions
+	switch predicateCall.Function {
+	case "startsWith":
+		return buildStartsWithPredicate(predicateCall, comp.IterVar)
+	case "endsWith":
+		return buildEndsWithPredicate(predicateCall, comp.IterVar)
+	case "contains":
+		return buildContainsPredicate(predicateCall, comp.IterVar)
+	default:
+		return nil, errors.Errorf("unsupported predicate function %q in comprehension (supported: startsWith, endsWith, contains)", predicateCall.Function)
+	}
+}
+
+// buildStartsWithPredicate extracts the pattern from t.startsWith("prefix").
+func buildStartsWithPredicate(call *exprv1.Expr_Call, iterVar string) (PredicateExpr, error) {
+	// Verify the target is the iteration variable
+	if target := call.Target.GetIdentExpr(); target == nil || target.GetName() != iterVar {
+		return nil, errors.Errorf("startsWith target must be the iteration variable %q", iterVar)
+	}
+
+	if len(call.Args) != 1 {
+		return nil, errors.New("startsWith expects exactly one argument")
+	}
+
+	prefix, err := getConstValue(call.Args[0])
+	if err != nil {
+		return nil, errors.Wrap(err, "startsWith argument must be a constant string")
+	}
+
+	prefixStr, ok := prefix.(string)
+	if !ok {
+		return nil, errors.New("startsWith argument must be a string")
+	}
+
+	return &StartsWithPredicate{Prefix: prefixStr}, nil
+}
+
+// buildEndsWithPredicate extracts the pattern from t.endsWith("suffix").
+func buildEndsWithPredicate(call *exprv1.Expr_Call, iterVar string) (PredicateExpr, error) {
+	if target := call.Target.GetIdentExpr(); target == nil || target.GetName() != iterVar {
+		return nil, errors.Errorf("endsWith target must be the iteration variable %q", iterVar)
+	}
+
+	if len(call.Args) != 1 {
+		return nil, errors.New("endsWith expects exactly one argument")
+	}
+
+	suffix, err := getConstValue(call.Args[0])
+	if err != nil {
+		return nil, errors.Wrap(err, "endsWith argument must be a constant string")
+	}
+
+	suffixStr, ok := suffix.(string)
+	if !ok {
+		return nil, errors.New("endsWith argument must be a string")
+	}
+
+	return &EndsWithPredicate{Suffix: suffixStr}, nil
+}
+
+// buildContainsPredicate extracts the pattern from t.contains("substring").
+func buildContainsPredicate(call *exprv1.Expr_Call, iterVar string) (PredicateExpr, error) {
+	if target := call.Target.GetIdentExpr(); target == nil || target.GetName() != iterVar {
+		return nil, errors.Errorf("contains target must be the iteration variable %q", iterVar)
+	}
+
+	if len(call.Args) != 1 {
+		return nil, errors.New("contains expects exactly one argument")
+	}
+
+	substring, err := getConstValue(call.Args[0])
+	if err != nil {
+		return nil, errors.Wrap(err, "contains argument must be a constant string")
+	}
+
+	substringStr, ok := substring.(string)
+	if !ok {
+		return nil, errors.New("contains argument must be a string")
+	}
+
+	return &ContainsPredicate{Substring: substringStr}, nil
+}
diff --git a/plugin/filter/render.go b/plugin/filter/render.go
@@ -74,6 +74,8 @@ func (r *renderer) renderCondition(cond Condition) (renderResult, error) {
 		return r.renderElementInCondition(c)
 	case *ContainsCondition:
 		return r.renderContainsCondition(c)
+	case *ListComprehensionCondition:
+		return r.renderListComprehension(c)
 	case *ConstantCondition:
 		if c.Value {
 			return renderResult{trivial: true}, nil
@@ -461,6 +463,101 @@ func (r *renderer) renderContainsCondition(cond *ContainsCondition) (renderResul
 	}
 }
 
+func (r *renderer) renderListComprehension(cond *ListComprehensionCondition) (renderResult, error) {
+	field, ok := r.schema.Field(cond.Field)
+	if !ok {
+		return renderResult{}, errors.Errorf("unknown field %q", cond.Field)
+	}
+
+	if field.Kind != FieldKindJSONList {
+		return renderResult{}, errors.Errorf("field %q is not a JSON list", cond.Field)
+	}
+
+	// Render based on predicate type
+	switch pred := cond.Predicate.(type) {
+	case *StartsWithPredicate:
+		return r.renderTagStartsWith(field, pred.Prefix, cond.Kind)
+	case *EndsWithPredicate:
+		return r.renderTagEndsWith(field, pred.Suffix, cond.Kind)
+	case *ContainsPredicate:
+		return r.renderTagContains(field, pred.Substring, cond.Kind)
+	default:
+		return renderResult{}, errors.Errorf("unsupported predicate type %T in comprehension", pred)
+	}
+}
+
+// renderTagStartsWith generates SQL for tags.exists(t, t.startsWith("prefix"))
+func (r *renderer) renderTagStartsWith(field Field, prefix string, _ ComprehensionKind) (renderResult, error) {
+	arrayExpr := jsonArrayExpr(r.dialect, field)
+
+	switch r.dialect {
+	case DialectSQLite, DialectMySQL:
+		// Match exact tag or tags with this prefix (hierarchical support)
+		exactMatch := r.buildJSONArrayLike(arrayExpr, fmt.Sprintf(`%%"%s"%%`, prefix))
+		prefixMatch := r.buildJSONArrayLike(arrayExpr, fmt.Sprintf(`%%"%s%%`, prefix))
+		condition := fmt.Sprintf("(%s OR %s)", exactMatch, prefixMatch)
+		return renderResult{sql: r.wrapWithNullCheck(arrayExpr, condition)}, nil
+
+	case DialectPostgres:
+		// Use PostgreSQL's powerful JSON operators
+		exactMatch := fmt.Sprintf("%s @> jsonb_build_array(%s::json)", arrayExpr, r.addArg(fmt.Sprintf(`"%s"`, prefix)))
+		prefixMatch := fmt.Sprintf("(%s)::text LIKE %s", arrayExpr, r.addArg(fmt.Sprintf(`%%"%s%%`, prefix)))
+		condition := fmt.Sprintf("(%s OR %s)", exactMatch, prefixMatch)
+		return renderResult{sql: r.wrapWithNullCheck(arrayExpr, condition)}, nil
+
+	default:
+		return renderResult{}, errors.Errorf("unsupported dialect %s", r.dialect)
+	}
+}
+
+// renderTagEndsWith generates SQL for tags.exists(t, t.endsWith("suffix"))
+func (r *renderer) renderTagEndsWith(field Field, suffix string, _ ComprehensionKind) (renderResult, error) {
+	arrayExpr := jsonArrayExpr(r.dialect, field)
+	pattern := fmt.Sprintf(`%%%s"%%`, suffix)
+
+	likeExpr := r.buildJSONArrayLike(arrayExpr, pattern)
+	return renderResult{sql: r.wrapWithNullCheck(arrayExpr, likeExpr)}, nil
+}
+
+// renderTagContains generates SQL for tags.exists(t, t.contains("substring"))
+func (r *renderer) renderTagContains(field Field, substring string, _ ComprehensionKind) (renderResult, error) {
+	arrayExpr := jsonArrayExpr(r.dialect, field)
+	pattern := fmt.Sprintf(`%%%s%%`, substring)
+
+	likeExpr := r.buildJSONArrayLike(arrayExpr, pattern)
+	return renderResult{sql: r.wrapWithNullCheck(arrayExpr, likeExpr)}, nil
+}
+
+// buildJSONArrayLike builds a LIKE expression for matching within a JSON array.
+// Returns the LIKE clause without NULL/empty checks.
+func (r *renderer) buildJSONArrayLike(arrayExpr, pattern string) string {
+	switch r.dialect {
+	case DialectSQLite, DialectMySQL:
+		return fmt.Sprintf("%s LIKE %s", arrayExpr, r.addArg(pattern))
+	case DialectPostgres:
+		return fmt.Sprintf("(%s)::text LIKE %s", arrayExpr, r.addArg(pattern))
+	default:
+		return ""
+	}
+}
+
+// wrapWithNullCheck wraps a condition with NULL and empty array checks.
+// This ensures we don't match against NULL or empty JSON arrays.
+func (r *renderer) wrapWithNullCheck(arrayExpr, condition string) string {
+	var nullCheck string
+	switch r.dialect {
+	case DialectSQLite:
+		nullCheck = fmt.Sprintf("%s IS NOT NULL AND %s != '[]'", arrayExpr, arrayExpr)
+	case DialectMySQL:
+		nullCheck = fmt.Sprintf("%s IS NOT NULL AND JSON_LENGTH(%s) > 0", arrayExpr, arrayExpr)
+	case DialectPostgres:
+		nullCheck = fmt.Sprintf("%s IS NOT NULL AND jsonb_array_length(%s) > 0", arrayExpr, arrayExpr)
+	default:
+		return condition
+	}
+	return fmt.Sprintf("(%s AND %s)", condition, nullCheck)
+}
+
 func (r *renderer) jsonBoolPredicate(field Field) (string, error) {
 	expr := jsonExtractExpr(r.dialect, field)
 	switch r.dialect {
diff --git a/store/test/memo_filter_comprehension_test.go b/store/test/memo_filter_comprehension_test.go
