From 78a135cfd1d08cab8ff1c9884c8c00b71f215b31 Mon Sep 17 00:00:00 2001 From: Chris Werner Rau Date: Wed, 11 Sep 2024 16:37:55 +0200 Subject: [PATCH 1/4] feat: implement partial apply this allows the user to create for example a HelmRelease for an operator and the corresponding resource in the same commit Signed-off-by: Chris Werner Rau --- api/v1/kustomization_types.go | 6 ++++ .../controller/kustomization_controller.go | 36 +++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/api/v1/kustomization_types.go b/api/v1/kustomization_types.go index ea6c92ae..c000829d 100644 --- a/api/v1/kustomization_types.go +++ b/api/v1/kustomization_types.go @@ -167,6 +167,12 @@ type KustomizationSpec struct { // Components specifies relative paths to specifications of other Components. // +optional Components []string `json:"components,omitempty"` + + // PartialApply instructs the controller to apply the kustomization partially + // if there are errors during the apply phase. + // +kubebuilder:default:=false + // +optional + PartialApply bool `json:"partialApply,omitempty"` } // CommonMetadata defines the common labels and annotations. diff --git a/internal/controller/kustomization_controller.go b/internal/controller/kustomization_controller.go index 830f4c22..02ed0222 100644 --- a/internal/controller/kustomization_controller.go +++ b/internal/controller/kustomization_controller.go @@ -29,6 +29,7 @@ import ( securejoin "github.com/cyphar/filepath-securejoin" "github.com/fluxcd/pkg/ssa/normalize" ssautil "github.com/fluxcd/pkg/ssa/utils" + "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -822,9 +823,38 @@ func (r *KustomizationReconciler) apply(ctx context.Context, // sort by kind, validate and apply all the others objects sort.Sort(ssa.SortableUnstructureds(resStage)) if len(resStage) > 0 { - changeSet, err := manager.ApplyAll(ctx, resStage, applyOpts) - if err != nil { - return false, nil, fmt.Errorf("%w\n%s", err, changeSetLog.String()) + changeSet := ssa.NewChangeSet() + if obj.Spec.PartialApply { + collectedChanges := make([]ssa.ChangeSetEntry, len(resStage)) + g := &errgroup.Group{} + g.SetLimit(r.ConcurrentSSA) + + for i, resource := range resStage { + g.Go(func() error { + changeSetEntry, err := manager.Apply(ctx, resource, applyOpts) + if err != nil { + return err + } else if changeSetEntry != nil { + collectedChanges[i] = *changeSetEntry + } + return nil + }) + } + if err := g.Wait(); err != nil { + for _, changeSetEntry := range collectedChanges { + if HasChanged(changeSetEntry.Action) { + changeSetLog.WriteString(changeSetEntry.String() + "\n") + } + } + return false, nil, fmt.Errorf("%w\n%s", err, changeSetLog.String()) + } + changeSet.Append(collectedChanges) + } else { + var err error + changeSet, err = manager.ApplyAll(ctx, resStage, applyOpts) + if err != nil { + return false, nil, fmt.Errorf("%w\n%s", err, changeSetLog.String()) + } } if changeSet != nil && len(changeSet.Entries) > 0 { From aa6a2b2da71c629c7c945e425d74f865b7a33d8a Mon Sep 17 00:00:00 2001 From: Chris Werner Rau Date: Thu, 12 Sep 2024 17:11:14 +0200 Subject: [PATCH 2/4] chore: add tests --- .../controller/kustomization_controller.go | 54 +++--- .../kustomization_partial_apply_test.go | 182 ++++++++++++++++++ 2 files changed, 211 insertions(+), 25 deletions(-) create mode 100644 internal/controller/kustomization_partial_apply_test.go diff --git a/internal/controller/kustomization_controller.go b/internal/controller/kustomization_controller.go index 02ed0222..34a7c261 100644 --- a/internal/controller/kustomization_controller.go +++ b/internal/controller/kustomization_controller.go @@ -421,7 +421,7 @@ func (r *KustomizationReconciler) reconcile( } // Validate and apply resources in stages. - drifted, changeSet, err := r.apply(ctx, resourceManager, obj, revision, objects) + drifted, changeSet, err, errDuringPartialApply := r.apply(ctx, resourceManager, obj, revision, objects) if err != nil { conditions.MarkFalse(obj, meta.ReadyCondition, meta.ReconciliationFailedReason, "%s", err) return err @@ -468,11 +468,15 @@ func (r *KustomizationReconciler) reconcile( // Set last applied revision. obj.Status.LastAppliedRevision = revision - // Mark the object as ready. - conditions.MarkTrue(obj, - meta.ReadyCondition, - meta.ReconciliationSucceededReason, - fmt.Sprintf("Applied revision: %s", revision)) + if errDuringPartialApply != nil { + conditions.MarkFalse(obj, meta.ReadyCondition, meta.ReconciliationFailedReason, fmt.Sprintf("Applied revision: %s with, err: %s", revision, errDuringPartialApply)) + } else { + // Mark the object as ready. + conditions.MarkTrue(obj, + meta.ReadyCondition, + meta.ReconciliationSucceededReason, + fmt.Sprintf("Applied revision: %s", revision)) + } return nil } @@ -658,11 +662,11 @@ func (r *KustomizationReconciler) apply(ctx context.Context, manager *ssa.ResourceManager, obj *kustomizev1.Kustomization, revision string, - objects []*unstructured.Unstructured) (bool, *ssa.ChangeSet, error) { + objects []*unstructured.Unstructured) (bool, *ssa.ChangeSet, error, error) { log := ctrl.LoggerFrom(ctx) if err := normalize.UnstructuredList(objects); err != nil { - return false, nil, err + return false, nil, err, nil } if cmeta := obj.Spec.CommonMetadata; cmeta != nil { @@ -752,7 +756,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context, if decryptor.IsEncryptedSecret(u) { return false, nil, fmt.Errorf("%s is SOPS encrypted, configuring decryption is required for this secret to be reconciled", - ssautil.FmtUnstructured(u)) + ssautil.FmtUnstructured(u)), nil } switch { @@ -772,7 +776,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context, if len(defStage) > 0 { changeSet, err := manager.ApplyAll(ctx, defStage, applyOpts) if err != nil { - return false, nil, err + return false, nil, err, nil } if changeSet != nil && len(changeSet.Entries) > 0 { @@ -789,7 +793,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context, Interval: 2 * time.Second, Timeout: obj.GetTimeout(), }); err != nil { - return false, nil, err + return false, nil, err, nil } } } @@ -798,7 +802,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context, if len(classStage) > 0 { changeSet, err := manager.ApplyAll(ctx, classStage, applyOpts) if err != nil { - return false, nil, err + return false, nil, err, nil } if changeSet != nil && len(changeSet.Entries) > 0 { @@ -815,11 +819,12 @@ func (r *KustomizationReconciler) apply(ctx context.Context, Interval: 2 * time.Second, Timeout: obj.GetTimeout(), }); err != nil { - return false, nil, err + return false, nil, err, nil } } } + var errorDuringPartialApply error = nil // sort by kind, validate and apply all the others objects sort.Sort(ssa.SortableUnstructureds(resStage)) if len(resStage) > 0 { @@ -833,36 +838,35 @@ func (r *KustomizationReconciler) apply(ctx context.Context, g.Go(func() error { changeSetEntry, err := manager.Apply(ctx, resource, applyOpts) if err != nil { + r.event(obj, revision, eventv1.EventSeverityError, fmt.Sprintf("error during apply for %v: %v", client.ObjectKeyFromObject(resource), err), nil) return err - } else if changeSetEntry != nil { - collectedChanges[i] = *changeSetEntry } + collectedChanges[i] = *changeSetEntry return nil }) } if err := g.Wait(); err != nil { - for _, changeSetEntry := range collectedChanges { - if HasChanged(changeSetEntry.Action) { - changeSetLog.WriteString(changeSetEntry.String() + "\n") - } - } - return false, nil, fmt.Errorf("%w\n%s", err, changeSetLog.String()) + errorDuringPartialApply = err } changeSet.Append(collectedChanges) } else { var err error changeSet, err = manager.ApplyAll(ctx, resStage, applyOpts) if err != nil { - return false, nil, fmt.Errorf("%w\n%s", err, changeSetLog.String()) + return false, nil, fmt.Errorf("%w\n%s", err, changeSetLog.String()), nil } } if changeSet != nil && len(changeSet.Entries) > 0 { resultSet.Append(changeSet.Entries) - log.Info("server-side apply completed", "output", changeSet.ToMap(), "revision", revision) + if errorDuringPartialApply != nil { + log.Info("server-side partial apply completed with error", "output", changeSet.ToMap(), "revision", revision, "err", errorDuringPartialApply) + } else { + log.Info("server-side apply completed", "output", changeSet.ToMap(), "revision", revision) + } for _, change := range changeSet.Entries { - if HasChanged(change.Action) { + if change.Action != "" && HasChanged(change.Action) { changeSetLog.WriteString(change.String() + "\n") } } @@ -875,7 +879,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context, r.event(obj, revision, eventv1.EventSeverityInfo, applyLog, nil) } - return applyLog != "", resultSet, nil + return applyLog != "", resultSet, nil, errorDuringPartialApply } func (r *KustomizationReconciler) checkHealth(ctx context.Context, diff --git a/internal/controller/kustomization_partial_apply_test.go b/internal/controller/kustomization_partial_apply_test.go new file mode 100644 index 00000000..f9c7a695 --- /dev/null +++ b/internal/controller/kustomization_partial_apply_test.go @@ -0,0 +1,182 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/testserver" + sourcev1 "github.com/fluxcd/source-controller/api/v1" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1" +) + +func TestKustomizationReconciler_PartialApply(t *testing.T) { + g := NewWithT(t) + id := "partial-apply-" + randStringRunes(5) + revision := "v1.0.0" + + err := createNamespace(id) + g.Expect(err).NotTo(HaveOccurred(), "failed to create test namespace") + + err = createKubeConfigSecret(id) + g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret") + + manifests := func(name string, data string) []testserver.File { + return []testserver.File{ + { + Name: "custom-resource.yaml", + Body: fmt.Sprintf(`--- +apiVersion: example.com/v1 +kind: CustomResource +metadata: + name: %[1]s +spec: + exampleField: "%[2]s" +`, name, data), + }, + { + Name: "secret.yaml", + Body: fmt.Sprintf(`--- +apiVersion: v1 +kind: Secret +metadata: + name: %[1]s +stringData: + key: "%[2]s" +`, name, data), + }, + } + } + + artifact, err := testServer.ArtifactFromFiles(manifests(id, randStringRunes(5))) + g.Expect(err).NotTo(HaveOccurred(), "failed to create artifact from files") + + repositoryName := types.NamespacedName{ + Name: fmt.Sprintf("partial-apply-%s", randStringRunes(5)), + Namespace: id, + } + + err = applyGitRepository(repositoryName, artifact, revision) + g.Expect(err).NotTo(HaveOccurred()) + + kustomizationKey := types.NamespacedName{ + Name: fmt.Sprintf("partial-apply-%s", randStringRunes(5)), + Namespace: id, + } + kustomization := &kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: kustomizationKey.Name, + Namespace: kustomizationKey.Namespace, + }, + Spec: kustomizev1.KustomizationSpec{ + Interval: metav1.Duration{Duration: reconciliationInterval}, + Path: "./", + KubeConfig: &meta.KubeConfigReference{ + SecretRef: meta.SecretKeyReference{ + Name: "kubeconfig", + }, + }, + SourceRef: kustomizev1.CrossNamespaceSourceReference{ + Name: repositoryName.Name, + Namespace: repositoryName.Namespace, + Kind: sourcev1.GitRepositoryKind, + }, + HealthChecks: []meta.NamespacedObjectKindReference{ + { + APIVersion: "v1", + Kind: "Secret", + Name: id, + Namespace: id, + }, + }, + TargetNamespace: id, + PartialApply: false, + }, + } + + g.Expect(k8sClient.Create(context.Background(), kustomization)).To(Succeed()) + + resultK := &kustomizev1.Kustomization{} + + t.Run("fails to apply resources", func(t *testing.T) { + artifact, err = testServer.ArtifactFromFiles(manifests(id, randStringRunes(5))) + g.Expect(err).NotTo(HaveOccurred()) + revision = "v1.0.0" + err = applyGitRepository(repositoryName, artifact, revision) + g.Expect(err).NotTo(HaveOccurred()) + + g.Eventually(func() bool { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + return isReconcileFailure(resultK) + }, timeout, time.Second).Should(BeTrue()) + logStatus(t, resultK) + + kstatusCheck.CheckErr(ctx, resultK) + + t.Run("emits dry-run error event", func(t *testing.T) { + events := getEvents(resultK.GetName(), map[string]string{"kustomize.toolkit.fluxcd.io/revision": revision}) + g.Expect(len(events) > 0).To(BeTrue()) + g.Expect(events[0].Type).To(BeIdenticalTo("Warning")) + g.Expect(events[0].Message).To(ContainSubstring("dry-run failed: no matches for kind")) + }) + }) + + t.Run("partially applies secret", func(t *testing.T) { + artifact, err = testServer.ArtifactFromFiles(manifests(id, randStringRunes(5))) + g.Expect(err).NotTo(HaveOccurred()) + revision = "v2.0.0" + err = applyGitRepository(repositoryName, artifact, revision) + g.Expect(err).NotTo(HaveOccurred()) + + g.Eventually(func() error { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + resultK.Spec.PartialApply = true + return k8sClient.Update(context.Background(), resultK) + }, timeout, time.Second).Should(BeNil()) + + g.Eventually(func() bool { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + hasCreatedSecret := !apierrors.IsNotFound(k8sClient.Get(context.Background(), types.NamespacedName{Name: id, Namespace: id}, &corev1.Secret{})) + return hasCreatedSecret && apimeta.IsStatusConditionFalse(resultK.Status.Conditions, meta.ReadyCondition) + }, timeout, time.Second).Should(BeTrue()) + logStatus(t, resultK) + + t.Run("emits partial apply error event", func(t *testing.T) { + events := getEvents(resultK.GetName(), map[string]string{"kustomize.toolkit.fluxcd.io/revision": revision}) + g.Expect(len(events) > 0).To(BeTrue()) + g.Expect(events[0].Type).To(BeIdenticalTo("Warning")) + g.Expect(events[0].Message).To(ContainSubstring("error during apply")) + g.Expect(events[0].Message).To(ContainSubstring("dry-run failed: no matches for kind")) + }) + + g.Expect(resultK.Status.Inventory.Entries).To(ContainElement(kustomizev1.ResourceRef{ID: fmt.Sprintf("%s_%s__Secret", id, id), Version: "v1"})) + + kstatusCheck.CheckErr(ctx, resultK) + }) +} From b559e883b1a6bbdb993eca29d34c01fb04b0ac70 Mon Sep 17 00:00:00 2001 From: Chris Werner Rau Date: Thu, 12 Sep 2024 17:11:46 +0200 Subject: [PATCH 3/4] chore: update CRD --- .../bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml index 16ed18b2..4f402708 100644 --- a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml +++ b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml @@ -231,6 +231,12 @@ spec: maxLength: 200 minLength: 1 type: string + partialApply: + default: false + description: |- + PartialApply instructs the controller to apply the kustomization partially + if there are errors during the apply phase. + type: boolean patches: description: |- Strategic merge and JSON patches, defined as inline YAML objects, From 7bab52b069534f47a6d9760eb0004dd0d1350f21 Mon Sep 17 00:00:00 2001 From: Chris Werner Rau Date: Thu, 12 Sep 2024 17:12:05 +0200 Subject: [PATCH 4/4] chore: update docs --- docs/api/v1/kustomize.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/api/v1/kustomize.md b/docs/api/v1/kustomize.md index 7a2281ad..bc067d89 100644 --- a/docs/api/v1/kustomize.md +++ b/docs/api/v1/kustomize.md @@ -380,6 +380,19 @@ resources. When enabled, the HealthChecks are ignored. Defaults to false.

Components specifies relative paths to specifications of other Components.

+ + +partialApply
+ +bool + + + +(Optional) +

PartialApply instructs the controller to apply the kustomization partially +if there are errors during the apply phase.

+ + @@ -888,6 +901,19 @@ resources. When enabled, the HealthChecks are ignored. Defaults to false.

Components specifies relative paths to specifications of other Components.

+ + +partialApply
+ +bool + + + +(Optional) +

PartialApply instructs the controller to apply the kustomization partially +if there are errors during the apply phase.

+ +