[pkg/ottl] Support dynamic pattern key in keep_keys and keep_matching_keys functions (open-telemetry#43729)

Co-authored-by: Evan Bradley <[email protected]>

Author: odubajDT

.chloggen/ottl-dynamic-keep-keys.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog)
+component: pkg/ottl
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Support paths and expressions as keys in `keep_keys` and `keep_matching_keys`
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [43555]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

pkg/ottl/e2e/e2e_test.go

@@ -64,6 +64,17 @@ func Test_e2e_editors(t *testing.T) {
 				tCtx.GetLogRecord().Attributes().Remove("conflict")
 			},
 		},
+		{
+			statement: `keep_matching_keys(attributes, Concat(["^", "http"], ""))`,
+			want: func(tCtx ottllog.TransformContext) {
+				tCtx.GetLogRecord().Attributes().Remove("flags")
+				tCtx.GetLogRecord().Attributes().Remove("total.string")
+				tCtx.GetLogRecord().Attributes().Remove("foo")
+				tCtx.GetLogRecord().Attributes().Remove("things")
+				tCtx.GetLogRecord().Attributes().Remove("conflict.conflict1")
+				tCtx.GetLogRecord().Attributes().Remove("conflict")
+			},
+		},
 		{
 			statement: `flatten(attributes)`,
 			want: func(tCtx ottllog.TransformContext) {
@@ -1225,6 +1236,14 @@ func Test_e2e_converters(t *testing.T) {
 				tCtx.GetLogRecord().Attributes().PutStr("test", `"`)
 			},
 		},
+		{
+			statement: `keep_keys(attributes["foo"], [Concat(["ba", "r"], "")])`,
+			want: func(tCtx ottllog.TransformContext) {
+				// keep_keys should see two arguments
+				m := tCtx.GetLogRecord().Attributes().PutEmptyMap("foo")
+				m.PutStr("bar", "pass")
+			},
+		},
 		{
 			statement: `keep_keys(attributes["foo"], ["\\", "bar"])`,
 			want: func(tCtx ottllog.TransformContext) {

pkg/ottl/ottlfuncs/func_keep_keys.go

@@ -14,7 +14,7 @@ import (
 
 type KeepKeysArguments[K any] struct {
 	Target ottl.PMapGetSetter[K]
-	Keys   []string
+	Keys   []ottl.StringGetter[K]
 }
 
 func NewKeepKeysFactory[K any]() ottl.Factory[K] {
@@ -31,17 +31,40 @@ func createKeepKeysFunction[K any](_ ottl.FunctionContext, oArgs ottl.Arguments)
 	return keepKeys(args.Target, args.Keys), nil
 }
 
-func keepKeys[K any](target ottl.PMapGetSetter[K], keys []string) ottl.ExprFunc[K] {
-	keySet := make(map[string]struct{}, len(keys))
+func keepKeys[K any](target ottl.PMapGetSetter[K], keys []ottl.StringGetter[K]) ottl.ExprFunc[K] {
+	// Check if all keys are literals and pre-build the key set if so
+	literalKeySet := make(map[string]struct{}, len(keys))
 	for _, key := range keys {
-		keySet[key] = struct{}{}
+		k, isLiteral := ottl.GetLiteralValue(key)
+		if !isLiteral {
+			literalKeySet = nil
+			break
+		}
+		literalKeySet[k] = struct{}{}
 	}
 
 	return func(ctx context.Context, tCtx K) (any, error) {
 		val, err := target.Get(ctx, tCtx)
 		if err != nil {
 			return nil, err
 		}
+
+		var keySet map[string]struct{}
+		if literalKeySet != nil {
+			// Use pre-built key set for literal keys
+			keySet = literalKeySet
+		} else {
+			// Build key set at runtime for dynamic keys
+			keySet = make(map[string]struct{}, len(keys))
+			for _, key := range keys {
+				k, err := key.Get(ctx, tCtx)
+				if err != nil {
+					return nil, err
+				}
+				keySet[k] = struct{}{}
+			}
+		}
+
 		val.RemoveIf(func(key string, _ pcommon.Value) bool {
 			_, ok := keySet[key]
 			return !ok

pkg/ottl/ottlfuncs/func_keep_keys_test.go

@@ -71,7 +71,17 @@ func Test_keepKeys(t *testing.T) {
 				},
 			}
 
-			exprFunc := keepKeys(target, tt.keys)
+			keys := make([]ottl.StringGetter[pcommon.Map], len(tt.keys))
+			for i, key := range tt.keys {
+				k := key
+				keys[i] = ottl.StandardStringGetter[pcommon.Map]{
+					Getter: func(_ context.Context, _ pcommon.Map) (any, error) {
+						return k, nil
+					},
+				}
+			}
+
+			exprFunc := keepKeys(target, keys)
 
 			_, err := exprFunc(nil, scenarioMap)
 			assert.NoError(t, err)
@@ -96,7 +106,13 @@ func Test_keepKeys_bad_input(t *testing.T) {
 		},
 	}
 
-	keys := []string{"anything"}
+	keys := []ottl.StringGetter[any]{
+		ottl.StandardStringGetter[any]{
+			Getter: func(_ context.Context, _ any) (any, error) {
+				return "anything", nil
+			},
+		},
+	}
 
 	exprFunc := keepKeys[any](target, keys)
 
@@ -114,7 +130,13 @@ func Test_keepKeys_get_nil(t *testing.T) {
 		},
 	}
 
-	keys := []string{"anything"}
+	keys := []ottl.StringGetter[any]{
+		ottl.StandardStringGetter[any]{
+			Getter: func(_ context.Context, _ any) (any, error) {
+				return "anything", nil
+			},
+		},
+	}
 
 	exprFunc := keepKeys[any](target, keys)
 	_, err := exprFunc(nil, nil)

pkg/ottl/ottlfuncs/func_keep_matching_keys.go

@@ -4,8 +4,6 @@
 package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottlfuncs"
 import (
 	"errors"
-	"fmt"
-	"regexp"
 
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"golang.org/x/net/context"
@@ -15,7 +13,7 @@ import (
 
 type KeepMatchingKeysArguments[K any] struct {
 	Target  ottl.PMapGetSetter[K]
-	Pattern string
+	Pattern ottl.StringGetter[K]
 }
 
 func NewKeepMatchingKeysFactory[K any]() ottl.Factory[K] {
@@ -32,20 +30,24 @@ func createKeepMatchingKeysFunction[K any](_ ottl.FunctionContext, oArgs ottl.Ar
 	return keepMatchingKeys(args.Target, args.Pattern)
 }
 
-func keepMatchingKeys[K any](target ottl.PMapGetSetter[K], pattern string) (ottl.ExprFunc[K], error) {
-	compiledPattern, err := regexp.Compile(pattern)
+func keepMatchingKeys[K any](target ottl.PMapGetSetter[K], pattern ottl.StringGetter[K]) (ottl.ExprFunc[K], error) {
+	compiledPattern, err := newDynamicRegex("keep_matching_keys", pattern)
 	if err != nil {
-		return nil, fmt.Errorf("the regex pattern provided to keep_matching_keys is not a valid pattern: %w", err)
+		return nil, err
 	}
-
 	return func(ctx context.Context, tCtx K) (any, error) {
+		cp, err := compiledPattern.compile(ctx, tCtx)
+		if err != nil {
+			return nil, err
+		}
+
 		val, err := target.Get(ctx, tCtx)
 		if err != nil {
 			return nil, err
 		}
 
 		val.RemoveIf(func(key string, _ pcommon.Value) bool {
-			return !compiledPattern.MatchString(key)
+			return !cp.MatchString(key)
 		})
 		return nil, target.Set(ctx, tCtx, val)
 	}, nil

pkg/ottl/ottlfuncs/func_keep_matching_keys_test.go

@@ -58,14 +58,6 @@ func Test_keepMatchingKeys(t *testing.T) {
 				return &m
 			},
 		},
-		{
-			name:    "invalid pattern",
-			pattern: "*",
-			want: func() *pcommon.Map {
-				return nil
-			},
-			wantError: true,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -87,7 +79,12 @@ func Test_keepMatchingKeys(t *testing.T) {
 				},
 			}
 
-			exprFunc, err := keepMatchingKeys(target, tt.pattern)
+			pattern := &ottl.StandardStringGetter[pcommon.Map]{
+				Getter: func(_ context.Context, _ pcommon.Map) (any, error) {
+					return tt.pattern, nil
+				},
+			}
+			exprFunc, err := keepMatchingKeys(target, pattern)
 
 			if tt.wantError {
 				assert.Error(t, err)
@@ -115,7 +112,37 @@ func Test_keepMatchingKeys_bad_input(t *testing.T) {
 		},
 	}
 
-	exprFunc, err := keepMatchingKeys[any](target, "anything")
+	pattern := &ottl.StandardStringGetter[any]{
+		Getter: func(_ context.Context, _ any) (any, error) {
+			return "anything", nil
+		},
+	}
+
+	exprFunc, err := keepMatchingKeys[any](target, pattern)
+	assert.NoError(t, err)
+
+	_, err = exprFunc(nil, input)
+	assert.Error(t, err)
+}
+
+func Test_keepMatchingKeys_invalid_pattern(t *testing.T) {
+	input := pcommon.NewValueInt(1)
+	target := &ottl.StandardPMapGetSetter[any]{
+		Getter: func(_ context.Context, tCtx any) (pcommon.Map, error) {
+			if v, ok := tCtx.(pcommon.Map); ok {
+				return v, nil
+			}
+			return pcommon.Map{}, errors.New("expected pcommon.Map")
+		},
+	}
+
+	pattern := &ottl.StandardStringGetter[any]{
+		Getter: func(_ context.Context, _ any) (any, error) {
+			return "*", nil
+		},
+	}
+
+	exprFunc, err := keepMatchingKeys[any](target, pattern)
 	assert.NoError(t, err)
 
 	_, err = exprFunc(nil, input)
@@ -132,7 +159,13 @@ func Test_keepMatchingKeys_get_nil(t *testing.T) {
 		},
 	}
 
-	exprFunc, err := keepMatchingKeys[any](target, "anything")
+	pattern := &ottl.StandardStringGetter[any]{
+		Getter: func(_ context.Context, _ any) (any, error) {
+			return "anything", nil
+		},
+	}
+
+	exprFunc, err := keepMatchingKeys[any](target, pattern)
 	assert.NoError(t, err)
 	_, err = exprFunc(nil, nil)
 	assert.Error(t, err)