diff --git a/internal/controller/sync/controller.go b/internal/controller/sync/controller.go index 314caff..dde9266 100644 --- a/internal/controller/sync/controller.go +++ b/internal/controller/sync/controller.go @@ -103,8 +103,7 @@ func Create( remoteDummy.SetGroupVersionKind(remoteGVK) // create the syncer that holds the meat&potatoes of the synchronization logic - mutator := mutation.NewMutator(pubRes.Spec.Mutation) - syncer, err := sync.NewResourceSyncer(log, localManager.GetClient(), virtualWorkspaceCluster.GetClient(), pubRes, localCRD, mutator, stateNamespace, agentName) + syncer, err := sync.NewResourceSyncer(log, localManager.GetClient(), virtualWorkspaceCluster.GetClient(), pubRes, localCRD, mutation.NewMutator, stateNamespace, agentName) if err != nil { return nil, fmt.Errorf("failed to create syncer: %w", err) } diff --git a/internal/mutation/mutation.go b/internal/mutation/mutation.go deleted file mode 100644 index ef1a566..0000000 --- a/internal/mutation/mutation.go +++ /dev/null @@ -1,154 +0,0 @@ -/* -Copyright 2025 The KCP Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mutation - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "html/template" - "regexp" - "strings" - - "github.com/Masterminds/sprig/v3" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" - - syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" -) - -func ApplyResourceMutations(value any, mutations []syncagentv1alpha1.ResourceMutation, ctx *TemplateMutationContext) (any, error) { - for _, mut := range mutations { - var err error - value, err = ApplyResourceMutation(value, mut, ctx) - if err != nil { - return nil, err - } - } - - return value, nil -} - -func ApplyResourceMutation(value any, mut syncagentv1alpha1.ResourceMutation, ctx *TemplateMutationContext) (any, error) { - // encode current value as JSON - encoded, err := json.Marshal(value) - if err != nil { - return nil, fmt.Errorf("failed to JSON encode value: %w", err) - } - - // apply mutation - jsonData, err := applyResourceMutationToJSON(string(encoded), mut, ctx) - if err != nil { - return nil, err - } - - // decode back - var result any - err = json.Unmarshal([]byte(jsonData), &result) - if err != nil { - return nil, fmt.Errorf("failed to decode JSON: %w", err) - } - - return result, nil -} - -func applyResourceMutationToJSON(jsonData string, mut syncagentv1alpha1.ResourceMutation, ctx *TemplateMutationContext) (string, error) { - switch { - case mut.Delete != nil: - return applyResourceDeleteMutation(jsonData, *mut.Delete) - case mut.Template != nil: - return applyResourceTemplateMutation(jsonData, *mut.Template, ctx) - case mut.Regex != nil: - return applyResourceRegexMutation(jsonData, *mut.Regex) - default: - return "", errors.New("must use either regex, template or delete mutation") - } -} - -func applyResourceDeleteMutation(jsonData string, mut syncagentv1alpha1.ResourceDeleteMutation) (string, error) { - jsonData, err := sjson.Delete(jsonData, mut.Path) - if err != nil { - return "", fmt.Errorf("failed to delete value @ %s: %w", mut.Path, err) - } - - return jsonData, nil -} - -func applyResourceRegexMutation(jsonData string, mut syncagentv1alpha1.ResourceRegexMutation) (string, error) { - if mut.Pattern == "" { - return sjson.Set(jsonData, mut.Path, mut.Replacement) - } - - // get the current value - value := gjson.Get(jsonData, mut.Path) - if !value.Exists() { - return "", fmt.Errorf("path %s did not match any element in the document", mut.Path) - } - - expr, err := regexp.Compile(mut.Pattern) - if err != nil { - return "", fmt.Errorf("invalid pattern %q: %w", mut.Pattern, err) - } - - // this does apply some coalescing, like turning numbers into strings - strVal := value.String() - replacement := expr.ReplaceAllString(strVal, mut.Replacement) - - return sjson.Set(jsonData, mut.Path, replacement) -} - -func templateFuncMap() template.FuncMap { - funcs := sprig.TxtFuncMap() - funcs["join"] = strings.Join - return funcs -} - -type TemplateMutationContext struct { - // Value is always set by this package to the value found in the document. - Value gjson.Result - - LocalObject map[string]any - RemoteObject map[string]any -} - -func applyResourceTemplateMutation(jsonData string, mut syncagentv1alpha1.ResourceTemplateMutation, ctx *TemplateMutationContext) (string, error) { - // get the current value - value := gjson.Get(jsonData, mut.Path) - if !value.Exists() { - return "", fmt.Errorf("path %s did not match any element in the document", mut.Path) - } - - tpl, err := template.New("mutation").Funcs(templateFuncMap()).Parse(mut.Template) - if err != nil { - return "", fmt.Errorf("failed to parse template %q: %w", mut.Template, err) - } - - if ctx == nil { - ctx = &TemplateMutationContext{} - } - ctx.Value = value - - var buf bytes.Buffer - if err := tpl.Execute(&buf, *ctx); err != nil { - return "", fmt.Errorf("failed to execute template %q: %w", mut.Template, err) - } - - replacement := strings.TrimSpace(buf.String()) - - return sjson.Set(jsonData, mut.Path, replacement) -} diff --git a/internal/mutation/mutation_test.go b/internal/mutation/mutation_test.go deleted file mode 100644 index cdcb574..0000000 --- a/internal/mutation/mutation_test.go +++ /dev/null @@ -1,216 +0,0 @@ -/* -Copyright 2025 The KCP Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mutation - -import ( - "encoding/json" - "testing" - - syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" -) - -func TestApplyResourceMutation(t *testing.T) { - testcases := []struct { - name string - inputData string - mutation syncagentv1alpha1.ResourceMutation - ctx *TemplateMutationContext - expected string - }{ - // regex - - { - name: "regex: replace one existing value", - inputData: `{"spec":{"secretName":"foo"}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Regex: &syncagentv1alpha1.ResourceRegexMutation{ - Path: "spec.secretName", - Pattern: "", - Replacement: "new-value", - }, - }, - expected: `{"spec":{"secretName":"new-value"}}`, - }, - { - name: "regex: rewrite one existing value", - inputData: `{"spec":{"secretName":"foo"}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Regex: &syncagentv1alpha1.ResourceRegexMutation{ - Path: "spec.secretName", - Pattern: "o", - Replacement: "u", - }, - }, - expected: `{"spec":{"secretName":"fuu"}}`, - }, - { - name: "regex: should support grouping", - inputData: `{"spec":{"secretName":"foo"}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Regex: &syncagentv1alpha1.ResourceRegexMutation{ - Path: "spec.secretName", - Pattern: "(f)oo", - Replacement: "oo$1", - }, - }, - expected: `{"spec":{"secretName":"oof"}}`, - }, - { - name: "regex: coalesces to strings", - inputData: `{"spec":{"aNumber":24}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Regex: &syncagentv1alpha1.ResourceRegexMutation{ - Path: "spec.aNumber", - Pattern: "4", - Replacement: "5", - }, - }, - expected: `{"spec":{"aNumber":"25"}}`, - }, - { - name: "regex: can change types", - inputData: `{"spec":{"aNumber":24}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Regex: &syncagentv1alpha1.ResourceRegexMutation{ - Path: "spec", - Replacement: "new-value", - }, - }, - expected: `{"spec":"new-value"}`, - }, - { - name: "regex: can change types /2", - inputData: `{"spec":{"aNumber":24}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Regex: &syncagentv1alpha1.ResourceRegexMutation{ - Path: "spec", - // Due to the string coalescing, this will turn the {aNumber:42} object - // into a string, of which we match every character and return it, - // effectively stringify-ing an object. - Pattern: "(.)", - Replacement: "$1", - }, - }, - expected: `{"spec":"{\"aNumber\":24}"}`, - }, - { - name: "regex: can empty values", - inputData: `{"spec":{"aNumber":24}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Regex: &syncagentv1alpha1.ResourceRegexMutation{ - Path: "spec", - Replacement: "", - }, - }, - expected: `{"spec":""}`, - }, - { - name: "regex: can empty values /2", - inputData: `{"spec":{"aNumber":24}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Regex: &syncagentv1alpha1.ResourceRegexMutation{ - Path: "spec", - Pattern: ".+", - Replacement: "", - }, - }, - expected: `{"spec":""}`, - }, - - // templates - - { - name: "template: empty template returns empty value", - inputData: `{"spec":{"secretName":"foo"}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Template: &syncagentv1alpha1.ResourceTemplateMutation{ - Path: "spec.secretName", - }, - }, - expected: `{"spec":{"secretName":""}}`, - }, - { - name: "template: can change value type", - inputData: `{"spec":{"secretName":"foo"}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Template: &syncagentv1alpha1.ResourceTemplateMutation{ - Path: "spec", - }, - }, - expected: `{"spec":""}`, - }, - { - name: "template: execute basic template", - inputData: `{"spec":{"secretName":"foo"}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Template: &syncagentv1alpha1.ResourceTemplateMutation{ - Path: "spec.secretName", - Template: `{{ upper .Value.String }}`, - }, - }, - expected: `{"spec":{"secretName":"FOO"}}`, - }, - - // delete - - { - name: "delete: can remove object keys", - inputData: `{"spec":{"secretName":"foo"}}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Delete: &syncagentv1alpha1.ResourceDeleteMutation{ - Path: "spec.secretName", - }, - }, - expected: `{"spec":{}}`, - }, - { - name: "delete: can remove array items", - inputData: `{"spec":[1,2,3]}`, - mutation: syncagentv1alpha1.ResourceMutation{ - Delete: &syncagentv1alpha1.ResourceDeleteMutation{ - Path: "spec.1", - }, - }, - expected: `{"spec":[1,3]}`, - }, - } - - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - // encode current value as JSON - var inputData any - if err := json.Unmarshal([]byte(testcase.inputData), &inputData); err != nil { - t.Fatalf("Failed to JSON encode input data: %v", err) - } - - mutated, err := ApplyResourceMutation(inputData, testcase.mutation, testcase.ctx) - if err != nil { - t.Fatalf("Function returned unexpected error: %v", err) - } - - result, err := json.Marshal(mutated) - if err != nil { - t.Fatalf("Failed to JSON encode output: %v", err) - } - - output := string(result) - if testcase.expected != output { - t.Errorf("Expected %q, but got %q.", testcase.expected, output) - } - }) - } -} diff --git a/internal/mutation/mutator.go b/internal/mutation/mutator.go index d3cc6f7..9744fa0 100644 --- a/internal/mutation/mutator.go +++ b/internal/mutation/mutator.go @@ -17,8 +17,10 @@ limitations under the License. package mutation import ( + "errors" "fmt" + "github.com/kcp-dev/api-syncagent/internal/mutation/transformer" syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -36,71 +38,85 @@ type Mutator interface { } type mutator struct { - spec *syncagentv1alpha1.ResourceMutationSpec + spec *transformer.AggregateTransformer + status *transformer.AggregateTransformer } var _ Mutator = &mutator{} // NewMutator creates a new mutator, which will apply the mutation rules to a synced object, in // both directions. A nil spec is supported and will simply make the mutator not do anything. -func NewMutator(spec *syncagentv1alpha1.ResourceMutationSpec) Mutator { - return &mutator{ - spec: spec, - } -} - -func (m *mutator) MutateSpec(toMutate *unstructured.Unstructured, otherObj *unstructured.Unstructured) (*unstructured.Unstructured, error) { - if m.spec == nil || m.spec.Spec == nil { - return toMutate, nil +func NewMutator(spec *syncagentv1alpha1.ResourceMutationSpec) (Mutator, error) { + if spec == nil { + return nil, nil } - ctx := &TemplateMutationContext{ - RemoteObject: toMutate.Object, - } - - if otherObj != nil { - ctx.LocalObject = otherObj.Object - } - - mutatedObj, err := ApplyResourceMutations(toMutate.Object, m.spec.Spec, ctx) + specAgg, err := createAggregatedTransformer(spec.Spec) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot create transformer for spec: %w", err) } - obj, ok := mutatedObj.(map[string]any) - if !ok { - return nil, fmt.Errorf("mutations did not yield an object, but %T", mutatedObj) + statusAgg, err := createAggregatedTransformer(spec.Status) + if err != nil { + return nil, fmt.Errorf("cannot create transformer for status: %w", err) } - toMutate.Object = obj - - return toMutate, nil + return &mutator{ + spec: specAgg, + status: statusAgg, + }, nil } -func (m *mutator) MutateStatus(toMutate *unstructured.Unstructured, otherObj *unstructured.Unstructured) (*unstructured.Unstructured, error) { - if m.spec == nil || m.spec.Status == nil { +func (m *mutator) MutateSpec(toMutate *unstructured.Unstructured, otherObj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + if m == nil { return toMutate, nil } - ctx := &TemplateMutationContext{ - LocalObject: toMutate.Object, - } + return m.spec.Apply(toMutate, otherObj) +} - if otherObj != nil { - ctx.RemoteObject = otherObj.Object +func (m *mutator) MutateStatus(toMutate *unstructured.Unstructured, otherObj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + if m == nil { + return toMutate, nil } - mutatedObj, err := ApplyResourceMutations(toMutate.Object, m.spec.Status, ctx) - if err != nil { - return nil, err - } + return m.status.Apply(toMutate, otherObj) +} - obj, ok := mutatedObj.(map[string]any) - if !ok { - return nil, fmt.Errorf("mutations did not yield an object, but %T", mutatedObj) +func createAggregatedTransformer(mutations []syncagentv1alpha1.ResourceMutation) (*transformer.AggregateTransformer, error) { + agg := transformer.NewAggregate() + + for _, mut := range mutations { + var ( + trans any + err error + ) + + switch { + case mut.Delete != nil: + trans, err = transformer.NewDelete(mut.Delete) + if err != nil { + return nil, err + } + + case mut.Regex != nil: + trans, err = transformer.NewRegex(mut.Regex) + if err != nil { + return nil, err + } + + case mut.Template != nil: + trans, err = transformer.NewTemplate(mut.Template) + if err != nil { + return nil, err + } + + default: + return nil, errors.New("no valid mutation mechanism provided") + } + + agg.Add(trans) } - toMutate.Object = obj - - return toMutate, nil + return agg, nil } diff --git a/internal/mutation/transformer/aggregate.go b/internal/mutation/transformer/aggregate.go new file mode 100644 index 0000000..d367199 --- /dev/null +++ b/internal/mutation/transformer/aggregate.go @@ -0,0 +1,87 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transformer + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +type objectTransformer interface { + Apply(toMutate *unstructured.Unstructured, otherObj *unstructured.Unstructured) (*unstructured.Unstructured, error) +} + +type jsonTransformer interface { + ApplyJSON(toMutate string, otherObj string) (string, error) +} + +// AggregateTransformer calls multiple other aggregates in sequence. A nil AggregateTransformer +// is supported and will simply not do anything. +type AggregateTransformer struct { + transformers []any +} + +func NewAggregate() *AggregateTransformer { + return &AggregateTransformer{ + transformers: []any{}, + } +} + +func (m *AggregateTransformer) Add(transformer any) { + m.transformers = append(m.transformers, transformer) +} + +func (m *AggregateTransformer) Apply(toMutate *unstructured.Unstructured, otherObj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + if m == nil { + return toMutate, nil + } + + for _, transformer := range m.transformers { + switch asserted := transformer.(type) { + case objectTransformer: + var err error + toMutate, err = asserted.Apply(toMutate, otherObj) + if err != nil { + return nil, err + } + + case jsonTransformer: + encodedToMutate, err := EncodeObject(toMutate) + if err != nil { + return nil, fmt.Errorf("failed to JSON encode object: %w", err) + } + + encodedOtherObj, err := EncodeObject(otherObj) + if err != nil { + return nil, fmt.Errorf("failed to JSON encode object: %w", err) + } + + mutated, err := asserted.ApplyJSON(encodedToMutate, encodedOtherObj) + if err != nil { + return nil, err + } + + toMutate, err = DecodeObject(mutated) + if err != nil { + return nil, err + } + } + } + + return toMutate, nil +} diff --git a/internal/mutation/transformer/delete.go b/internal/mutation/transformer/delete.go new file mode 100644 index 0000000..eaed292 --- /dev/null +++ b/internal/mutation/transformer/delete.go @@ -0,0 +1,44 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transformer + +import ( + "fmt" + + "github.com/tidwall/sjson" + + syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" +) + +type deleteTransformer struct { + mut *syncagentv1alpha1.ResourceDeleteMutation +} + +func NewDelete(mut *syncagentv1alpha1.ResourceDeleteMutation) (*deleteTransformer, error) { + return &deleteTransformer{ + mut: mut, + }, nil +} + +func (m *deleteTransformer) ApplyJSON(toMutate string, _ string) (string, error) { + jsonData, err := sjson.Delete(toMutate, m.mut.Path) + if err != nil { + return "", fmt.Errorf("failed to delete value @ %s: %w", m.mut.Path, err) + } + + return jsonData, nil +} diff --git a/internal/mutation/transformer/delete_test.go b/internal/mutation/transformer/delete_test.go new file mode 100644 index 0000000..f764728 --- /dev/null +++ b/internal/mutation/transformer/delete_test.go @@ -0,0 +1,67 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transformer + +import ( + "testing" + + syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" +) + +func TestDelete(t *testing.T) { + testcases := []struct { + name string + inputData string + mutation syncagentv1alpha1.ResourceDeleteMutation + expected string + }{ + { + name: "can remove object keys", + inputData: `{"spec":{"secretName":"foo"}}`, + mutation: syncagentv1alpha1.ResourceDeleteMutation{ + Path: "spec.secretName", + }, + expected: `{"spec":{}}`, + }, + { + name: "can remove array items", + inputData: `{"spec":[1,2,3]}`, + mutation: syncagentv1alpha1.ResourceDeleteMutation{ + Path: "spec.1", + }, + expected: `{"spec":[1,3]}`, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + transformer, err := NewDelete(&testcase.mutation) + if err != nil { + t.Fatalf("Failed to create transformer: %v", err) + } + + transformed, err := transformer.ApplyJSON(testcase.inputData, "") + if err != nil { + t.Fatalf("Failed to transform: %v", err) + } + + if testcase.expected != transformed { + t.Errorf("Expected %q, but got %q.", testcase.expected, transformed) + } + }) + } +} diff --git a/internal/mutation/transformer/json.go b/internal/mutation/transformer/json.go new file mode 100644 index 0000000..f767515 --- /dev/null +++ b/internal/mutation/transformer/json.go @@ -0,0 +1,43 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transformer + +import ( + "encoding/json" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func EncodeObject(obj *unstructured.Unstructured) (string, error) { + encoded, err := json.Marshal(obj.Object) + if err != nil { + return "", err + } + + return string(encoded), nil +} + +func DecodeObject(encoded string) (*unstructured.Unstructured, error) { + result := map[string]any{} + if err := json.Unmarshal([]byte(encoded), &result); err != nil { + return nil, err + } + + return &unstructured.Unstructured{ + Object: result, + }, nil +} diff --git a/internal/mutation/transformer/regex.go b/internal/mutation/transformer/regex.go new file mode 100644 index 0000000..543cc52 --- /dev/null +++ b/internal/mutation/transformer/regex.go @@ -0,0 +1,63 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transformer + +import ( + "fmt" + "regexp" + + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + + syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" +) + +type regexTransformer struct { + mut *syncagentv1alpha1.ResourceRegexMutation + expr *regexp.Regexp +} + +func NewRegex(mut *syncagentv1alpha1.ResourceRegexMutation) (*regexTransformer, error) { + var ( + expr *regexp.Regexp + err error + ) + + if mut.Pattern != "" { + expr, err = regexp.Compile(mut.Pattern) + if err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", mut.Pattern, err) + } + } + + return ®exTransformer{ + mut: mut, + expr: expr, + }, nil +} + +func (m *regexTransformer) ApplyJSON(toMutate string, _ string) (string, error) { + if m.expr == nil { + return sjson.Set(toMutate, m.mut.Path, m.mut.Replacement) + } + + // this does apply some coalescing, like turning numbers into strings + strVal := gjson.Get(toMutate, m.mut.Path).String() + replacement := m.expr.ReplaceAllString(strVal, m.mut.Replacement) + + return sjson.Set(toMutate, m.mut.Path, replacement) +} diff --git a/internal/mutation/transformer/regex_test.go b/internal/mutation/transformer/regex_test.go new file mode 100644 index 0000000..aadcc51 --- /dev/null +++ b/internal/mutation/transformer/regex_test.go @@ -0,0 +1,132 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transformer + +import ( + "testing" + + syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" +) + +func TestRegex(t *testing.T) { + testcases := []struct { + name string + inputData string + mutation syncagentv1alpha1.ResourceRegexMutation + expected string + }{ + { + name: "replace one existing value", + inputData: `{"spec":{"secretName":"foo"}}`, + mutation: syncagentv1alpha1.ResourceRegexMutation{ + Path: "spec.secretName", + Pattern: "", + Replacement: "new-value", + }, + expected: `{"spec":{"secretName":"new-value"}}`, + }, + { + name: "rewrite one existing value", + inputData: `{"spec":{"secretName":"foo"}}`, + mutation: syncagentv1alpha1.ResourceRegexMutation{ + Path: "spec.secretName", + Pattern: "o", + Replacement: "u", + }, + expected: `{"spec":{"secretName":"fuu"}}`, + }, + { + name: "should support grouping", + inputData: `{"spec":{"secretName":"foo"}}`, + mutation: syncagentv1alpha1.ResourceRegexMutation{ + Path: "spec.secretName", + Pattern: "(f)oo", + Replacement: "oo$1", + }, + expected: `{"spec":{"secretName":"oof"}}`, + }, + { + name: "coalesces to strings", + inputData: `{"spec":{"aNumber":24}}`, + mutation: syncagentv1alpha1.ResourceRegexMutation{ + Path: "spec.aNumber", + Pattern: "4", + Replacement: "5", + }, + expected: `{"spec":{"aNumber":"25"}}`, + }, + { + name: "can change types", + inputData: `{"spec":{"aNumber":24}}`, + mutation: syncagentv1alpha1.ResourceRegexMutation{ + Path: "spec", + Replacement: "new-value", + }, + expected: `{"spec":"new-value"}`, + }, + { + name: "can change types /2", + inputData: `{"spec":{"aNumber":24}}`, + mutation: syncagentv1alpha1.ResourceRegexMutation{ + Path: "spec", + // Due to the string coalescing, this will turn the {aNumber:42} object + // into a string, of which we match every character and return it, + // effectively stringify-ing an object. + Pattern: "(.)", + Replacement: "$1", + }, + expected: `{"spec":"{\"aNumber\":24}"}`, + }, + { + name: "can empty values", + inputData: `{"spec":{"aNumber":24}}`, + mutation: syncagentv1alpha1.ResourceRegexMutation{ + Path: "spec", + Replacement: "", + }, + expected: `{"spec":""}`, + }, + { + name: "can empty values /2", + inputData: `{"spec":{"aNumber":24}}`, + mutation: syncagentv1alpha1.ResourceRegexMutation{ + Path: "spec", + Pattern: ".+", + Replacement: "", + }, + expected: `{"spec":""}`, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + transformer, err := NewRegex(&testcase.mutation) + if err != nil { + t.Fatalf("Failed to create transformer: %v", err) + } + + transformed, err := transformer.ApplyJSON(testcase.inputData, "") + if err != nil { + t.Fatalf("Failed to transform: %v", err) + } + + if testcase.expected != transformed { + t.Errorf("Expected %q, but got %q.", testcase.expected, transformed) + } + }) + } +} diff --git a/internal/mutation/transformer/template.go b/internal/mutation/transformer/template.go new file mode 100644 index 0000000..0ac24da --- /dev/null +++ b/internal/mutation/transformer/template.go @@ -0,0 +1,87 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transformer + +import ( + "bytes" + "fmt" + "html/template" + "strings" + + "github.com/Masterminds/sprig/v3" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + + syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +type templateTransformer struct { + path string + tpl *template.Template +} + +func NewTemplate(mut *syncagentv1alpha1.ResourceTemplateMutation) (*templateTransformer, error) { + tpl, err := template.New("mutation").Funcs(sprig.TxtFuncMap()).Parse(mut.Template) + if err != nil { + return nil, fmt.Errorf("failed to parse template %q: %w", mut.Template, err) + } + + return &templateTransformer{ + path: mut.Path, + tpl: tpl, + }, nil +} + +type templateMutationContext struct { + // Value is always set by this package to the value found in the document. + Value gjson.Result + + LocalObject map[string]any + RemoteObject map[string]any +} + +func (m *templateTransformer) Apply(toMutate *unstructured.Unstructured, otherObj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + encoded, err := EncodeObject(toMutate) + if err != nil { + return nil, fmt.Errorf("failed to JSON encode object: %w", err) + } + + ctx := templateMutationContext{ + Value: gjson.Get(encoded, m.path), + LocalObject: toMutate.Object, + } + + if otherObj != nil { + ctx.RemoteObject = otherObj.Object + } + + var buf bytes.Buffer + if err := m.tpl.Execute(&buf, ctx); err != nil { + return nil, fmt.Errorf("failed to execute template: %w", err) + } + + replacement := strings.TrimSpace(buf.String()) + + updated, err := sjson.Set(encoded, m.path, replacement) + if err != nil { + return nil, fmt.Errorf("failed to set updated value: %w", err) + } + + return DecodeObject(updated) +} diff --git a/internal/mutation/transformer/template_test.go b/internal/mutation/transformer/template_test.go new file mode 100644 index 0000000..09d990c --- /dev/null +++ b/internal/mutation/transformer/template_test.go @@ -0,0 +1,117 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transformer + +import ( + "testing" + + "github.com/kcp-dev/api-syncagent/internal/test/diff" + syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" + "github.com/kcp-dev/api-syncagent/test/utils" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestTemplate(t *testing.T) { + commonInputObject := utils.YAMLToUnstructured(t, ` +apiVersion: kcp.example.com/v1 +kind: CronTab +metadata: + namespace: default + name: my-crontab +spec: + cronSpec: '* * *' + image: ubuntu:latest +`) + + testcases := []struct { + name string + inputData *unstructured.Unstructured + otherObj *unstructured.Unstructured + mutation syncagentv1alpha1.ResourceTemplateMutation + expected *unstructured.Unstructured + }{ + { + name: "empty template returns empty value", + inputData: commonInputObject, + mutation: syncagentv1alpha1.ResourceTemplateMutation{ + Path: "spec.cronSpec", + }, + expected: utils.YAMLToUnstructured(t, ` +apiVersion: kcp.example.com/v1 +kind: CronTab +metadata: + namespace: default + name: my-crontab +spec: + cronSpec: '' + image: ubuntu:latest +`), + }, + { + name: "can change value type", + inputData: commonInputObject, + mutation: syncagentv1alpha1.ResourceTemplateMutation{ + Path: "spec", + }, + expected: utils.YAMLToUnstructured(t, ` +apiVersion: kcp.example.com/v1 +kind: CronTab +metadata: + namespace: default + name: my-crontab +spec: '' +`), + }, + { + name: "execute basic template", + inputData: commonInputObject, + mutation: syncagentv1alpha1.ResourceTemplateMutation{ + Path: "spec.image", + Template: `{{ upper .Value.String }}`, + }, + expected: utils.YAMLToUnstructured(t, ` +apiVersion: kcp.example.com/v1 +kind: CronTab +metadata: + namespace: default + name: my-crontab +spec: + cronSpec: '* * *' + image: UBUNTU:LATEST +`), + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + transformer, err := NewTemplate(&testcase.mutation) + if err != nil { + t.Fatalf("Failed to create transformer: %v", err) + } + + transformed, err := transformer.Apply(testcase.inputData, testcase.otherObj) + if err != nil { + t.Fatalf("Failed to transform: %v", err) + } + + if changes := diff.ObjectDiff(testcase.expected, transformed); changes != "" { + t.Errorf("Did not get expected object:\n\n%s", changes) + } + }) + } +} diff --git a/internal/sync/syncer.go b/internal/sync/syncer.go index 4cf239f..98dafe6 100644 --- a/internal/sync/syncer.go +++ b/internal/sync/syncer.go @@ -45,7 +45,9 @@ type ResourceSyncer struct { destDummy *unstructured.Unstructured - mutator mutation.Mutator + // cached mutators (for those transformers that are expensive to compile, like CEL) + primaryMutator mutation.Mutator + relatedMutators map[string]mutation.Mutator agentName string @@ -53,13 +55,15 @@ type ResourceSyncer struct { newObjectStateStore newObjectStateStoreFunc } +type MutatorCreatorFunc func(*syncagentv1alpha1.ResourceMutationSpec) (mutation.Mutator, error) + func NewResourceSyncer( log *zap.SugaredLogger, localClient ctrlruntimeclient.Client, remoteClient ctrlruntimeclient.Client, pubRes *syncagentv1alpha1.PublishedResource, localCRD *apiextensionsv1.CustomResourceDefinition, - mutator mutation.Mutator, + mutatorCreator MutatorCreatorFunc, stateNamespace string, agentName string, ) (*ResourceSyncer, error) { @@ -94,6 +98,21 @@ func NewResourceSyncer( } } + primaryMutator, err := mutatorCreator(pubRes.Spec.Mutation) + if err != nil { + return nil, fmt.Errorf("failed to create primary object mutator: %w", err) + } + + relatedMutators := map[string]mutation.Mutator{} + for _, rr := range pubRes.Spec.Related { + mutator, err := mutatorCreator(rr.Mutation) + if err != nil { + return nil, fmt.Errorf("failed to create related object %q mutator: %w", rr.Identifier, err) + } + + relatedMutators[rr.Identifier] = mutator + } + return &ResourceSyncer{ log: log.With("local-gvk", localGVK, "remote-gvk", remoteGVK), localClient: localClient, @@ -102,7 +121,8 @@ func NewResourceSyncer( localCRD: localCRD, subresources: subresources, destDummy: localDummy, - mutator: mutator, + primaryMutator: primaryMutator, + relatedMutators: relatedMutators, agentName: agentName, newObjectStateStore: newKubernetesStateStoreCreator(stateNamespace), }, nil @@ -162,7 +182,7 @@ func (s *ResourceSyncer) Process(ctx Context, remoteObj *unstructured.Unstructur // in kcp is deleted blockSourceDeletion: true, // use the configured mutations from the PublishedResource - mutator: s.mutator, + mutator: s.primaryMutator, // make sure the syncer can remember the current state of any object stateStore: stateStore, // For the main resource, we need to store metadata on the destination copy diff --git a/internal/sync/syncer_related.go b/internal/sync/syncer_related.go index b2a427d..2092cf5 100644 --- a/internal/sync/syncer_related.go +++ b/internal/sync/syncer_related.go @@ -27,7 +27,6 @@ import ( "github.com/tidwall/gjson" "go.uber.org/zap" - "github.com/kcp-dev/api-syncagent/internal/mutation" "github.com/kcp-dev/api-syncagent/internal/sync/templating" syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" @@ -142,7 +141,7 @@ func (s *ResourceSyncer) processRelatedResource(log *zap.SugaredLogger, stateSto // sure we can clean up properly blockSourceDeletion: relRes.Origin == "kcp", // apply mutation rules configured for the related resource - mutator: mutation.NewMutator(relRes.Mutation), + mutator: s.relatedMutators[relRes.Identifier], // we never want to store sync-related metadata inside kcp metadataOnDestination: false, } diff --git a/internal/sync/syncer_test.go b/internal/sync/syncer_test.go index e34dbbf..a283540 100644 --- a/internal/sync/syncer_test.go +++ b/internal/sync/syncer_test.go @@ -26,6 +26,7 @@ import ( "github.com/kcp-dev/logicalcluster/v3" "go.uber.org/zap" + "github.com/kcp-dev/api-syncagent/internal/mutation" dummyv1alpha1 "github.com/kcp-dev/api-syncagent/internal/sync/apis/dummy/v1alpha1" "github.com/kcp-dev/api-syncagent/internal/test/diff" syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" @@ -899,7 +900,9 @@ func TestSyncerProcessingSingleResourceWithoutStatus(t *testing.T) { remoteClient, testcase.pubRes, testcase.localCRD, - nil, + func(rms *syncagentv1alpha1.ResourceMutationSpec) (mutation.Mutator, error) { + return nil, nil + }, stateNamespace, "textor-the-doctor", ) @@ -1205,7 +1208,9 @@ func TestSyncerProcessingSingleResourceWithStatus(t *testing.T) { remoteClient, testcase.pubRes, testcase.localCRD, - nil, + func(rms *syncagentv1alpha1.ResourceMutationSpec) (mutation.Mutator, error) { + return nil, nil + }, stateNamespace, "textor-the-doctor", ) diff --git a/test/e2e/sync/primary_test.go b/test/e2e/sync/primary_test.go index 0958f19..8214f98 100644 --- a/test/e2e/sync/primary_test.go +++ b/test/e2e/sync/primary_test.go @@ -35,12 +35,9 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer/yaml" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - yamlutil "k8s.io/apimachinery/pkg/util/yaml" ctrlruntime "sigs.k8s.io/controller-runtime" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/kontext" @@ -105,7 +102,7 @@ func TestSyncSimpleObject(t *testing.T) { // create a Crontab object in a team workspace t.Log("Creating CronTab in kcp…") - crontab := yamlToUnstructured(t, ` + crontab := utils.YAMLToUnstructured(t, ` apiVersion: kcp.example.com/v1 kind: CronTab metadata: @@ -195,7 +192,7 @@ func TestSyncSimpleObjectOldNaming(t *testing.T) { // create a Crontab object in a team workspace t.Log("Creating CronTab in kcp…") - crontab := yamlToUnstructured(t, ` + crontab := utils.YAMLToUnstructured(t, ` apiVersion: kcp.example.com/v1 kind: CronTab metadata: @@ -294,7 +291,7 @@ spec: teamCtx := kontext.WithCluster(ctx, logicalcluster.Name(fmt.Sprintf("root:%s:%s", orgWorkspace, team))) utils.WaitForBoundAPI(t, teamCtx, kcpClient, crontabsGVR) - if err := kcpClient.Create(teamCtx, yamlToUnstructured(t, crontabYAML)); err != nil { + if err := kcpClient.Create(teamCtx, utils.YAMLToUnstructured(t, crontabYAML)); err != nil { t.Fatalf("Failed to create %s's CronTab in kcp: %v", team, err) } } @@ -379,7 +376,7 @@ func TestLocalChangesAreKept(t *testing.T) { // create a Crontab object in a team workspace t.Log("Creating CronTab in kcp…") - crontab := yamlToUnstructured(t, ` + crontab := utils.YAMLToUnstructured(t, ` apiVersion: kcp.example.com/v1 kind: CronTab metadata: @@ -507,25 +504,6 @@ spec: } } -func yamlToUnstructured(t *testing.T, data string) *unstructured.Unstructured { - t.Helper() - - decoder := yamlutil.NewYAMLOrJSONDecoder(strings.NewReader(data), 100) - - var rawObj runtime.RawExtension - if err := decoder.Decode(&rawObj); err != nil { - t.Fatalf("Failed to decode: %v", err) - } - - obj, _, err := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).Decode(rawObj.Raw, nil, nil) - unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) - if err != nil { - t.Fatal(err) - } - - return &unstructured.Unstructured{Object: unstructuredMap} -} - func TestResourceFilter(t *testing.T) { const ( apiExportName = "kcp.example.com" @@ -592,7 +570,7 @@ func TestResourceFilter(t *testing.T) { // create two Crontab objects in a team workspace t.Log("Creating CronTab in kcp…") - ignoredCrontab := yamlToUnstructured(t, ` + ignoredCrontab := utils.YAMLToUnstructured(t, ` apiVersion: kcp.example.com/v1 kind: CronTab metadata: @@ -606,7 +584,7 @@ spec: t.Fatalf("Failed to create CronTab in kcp: %v", err) } - includedCrontab := yamlToUnstructured(t, ` + includedCrontab := utils.YAMLToUnstructured(t, ` apiVersion: kcp.example.com/v1 kind: CronTab metadata: @@ -713,7 +691,7 @@ func TestSyncingOverlyLongNames(t *testing.T) { } t.Log("Creating CronTab in kcp…") - ignoredCrontab := yamlToUnstructured(t, ` + ignoredCrontab := utils.YAMLToUnstructured(t, ` apiVersion: kcp.example.com/v1 kind: CronTab metadata: diff --git a/test/utils/utils.go b/test/utils/utils.go index f48a7f1..845ef66 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -21,6 +21,7 @@ import ( "net/url" "os" "regexp" + "strings" "testing" syncagentv1alpha1 "github.com/kcp-dev/api-syncagent/sdk/apis/syncagent/v1alpha1" @@ -33,6 +34,8 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/scale/scheme" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -184,3 +187,21 @@ func ToUnstructured(t *testing.T, obj any) *unstructured.Unstructured { return &unstructured.Unstructured{Object: raw} } + +func YAMLToUnstructured(t *testing.T, data string) *unstructured.Unstructured { + t.Helper() + + decoder := yamlutil.NewYAMLOrJSONDecoder(strings.NewReader(data), 100) + + var rawObj runtime.RawExtension + if err := decoder.Decode(&rawObj); err != nil { + t.Fatalf("Failed to decode: %v", err) + } + + obj, _, err := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme).Decode(rawObj.Raw, nil, nil) + if err != nil { + t.Fatalf("Failed to decode: %v", err) + } + + return ToUnstructured(t, obj) +}