Merge pull request #55 from fluxcd/suspect-and-force

Suspend and force mechanisms

Author: squaremo

api/v1alpha1/imageupdateautomation_types.go

@@ -40,6 +40,11 @@ type ImageUpdateAutomationSpec struct {
 	// Commit specifies how to commit to the git repo
 	// +required
 	Commit CommitSpec `json:"commit"`
+
+	// Suspend tells the controller to not run this automation, until
+	// it is unset (or set to false). Defaults to false.
+	// +optional
+	Suspend bool `json:"suspend,omitempty"`
 }
 
 type GitCheckoutSpec struct {
@@ -97,7 +102,8 @@ type ImageUpdateAutomationStatus struct {
 	// +optional
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 	// +optional
-	Conditions []metav1.Condition `json:"conditions,omitempty"`
+	Conditions                  []metav1.Condition `json:"conditions,omitempty"`
+	meta.ReconcileRequestStatus `json:",inline"`
 }
 
 const (

api/v1alpha1/zz_generated.deepcopy.go

@@ -86,6 +86,10 @@ spec:
                 description: RunInterval gives a lower bound for how often the automation
                   run should be attempted. Otherwise it will default.
                 type: string
+              suspend:
+                description: Suspend tells the controller to not run this automation,
+                  until it is unset (or set to false). Defaults to false.
+                type: boolean
               update:
                 description: Update gives the specification for how to update the
                   files in the repository
@@ -187,6 +191,10 @@ spec:
                   made).
                 format: date-time
                 type: string
+              lastHandledReconcileAt:
+                description: LastHandledReconcileAt holds the value of the most recent
+                  reconcile request value, so a change can be detected.
+                type: string
               observedGeneration:
                 format: int64
                 type: integer

config/crd/bases/image.toolkit.fluxcd.io_imageupdateautomations.yaml

@@ -87,6 +87,20 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	if auto.Spec.Suspend {
+		log.Info("ImageUpdateAutomation is suspended, skipping automation run")
+		return ctrl.Result{}, nil
+	}
+
+	// whatever else happens, we've now "seen" the reconcile
+	// annotation if it's there
+	if token, ok := meta.ReconcileAnnotationValue(auto.GetAnnotations()); ok {
+		auto.Status.SetLastHandledReconcileRequest(token)
+		if err := r.Status().Update(ctx, &auto); err != nil {
+			return ctrl.Result{Requeue: true}, err
+		}
+	}
+
 	// failWithError is a helper for bailing on the reconciliation.
 	failWithError := func(err error) (ctrl.Result, error) {
 		r.event(auto, events.EventSeverityError, err.Error())

controllers/imageupdateautomation_controller.go

@@ -44,6 +44,7 @@ import (
 var cfg *rest.Config
 var k8sClient client.Client
 var k8sManager ctrl.Manager
+var imageAutoReconciler *ImageUpdateAutomationReconciler
 var testEnv *envtest.Environment
 
 func TestAPIs(t *testing.T) {
@@ -81,19 +82,22 @@ var _ = BeforeSuite(func(done Done) {
 	})
 	Expect(err).ToNot(HaveOccurred())
 
-	err = (&ImageUpdateAutomationReconciler{
+	imageAutoReconciler = &ImageUpdateAutomationReconciler{
 		Client: k8sManager.GetClient(),
 		Log:    ctrl.Log.WithName("controllers").WithName("ImageUpdateAutomation"),
 		Scheme: scheme.Scheme,
-	}).SetupWithManager(k8sManager)
-	Expect(err).ToNot(HaveOccurred())
+	}
+	Expect(imageAutoReconciler.SetupWithManager(k8sManager)).To(Succeed())
 
 	go func() {
 		err = k8sManager.Start(ctrl.SetupSignalHandler())
 		Expect(err).ToNot(HaveOccurred())
 	}()
 
-	k8sClient = k8sManager.GetClient()
+	// Specifically an uncached client. Use <reconciler>.Get if you
+	// want to see what the reconcilers see.
+	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+	Expect(err).ToNot(HaveOccurred())
 	Expect(k8sClient).ToNot(BeNil())
 
 	close(done)

controllers/suite_test.go

@@ -36,10 +36,14 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/otiai10/copy"
 	corev1 "k8s.io/api/core/v1"
+	//	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	imagev1_reflect "github.com/fluxcd/image-reflector-controller/api/v1alpha1"
+	"github.com/fluxcd/pkg/apis/meta"
 	"github.com/fluxcd/pkg/gittestserver"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
 
@@ -185,29 +189,9 @@ var _ = Describe("ImageUpdateAutomation", func() {
 			BeforeEach(func() {
 				// Insert a setter reference into the deployment file,
 				// before creating the automation object itself.
-				tmp, err := ioutil.TempDir("", "gotest-imageauto-setters")
-				Expect(err).ToNot(HaveOccurred())
-				defer os.RemoveAll(tmp)
-				repo, err := git.PlainClone(tmp, false, &git.CloneOptions{
-					URL:           repoURL,
-					ReferenceName: plumbing.NewBranchReferenceName(defaultBranch),
+				commitInRepo(repoURL, "Install setter marker", func(tmp string) {
+					replaceMarker(tmp, policyKey)
 				})
-				Expect(err).ToNot(HaveOccurred())
-
-				replaceMarker(tmp, policyKey)
-				worktree, err := repo.Worktree()
-				Expect(err).ToNot(HaveOccurred())
-				_, err = worktree.Add("deploy.yaml")
-				Expect(err).ToNot(HaveOccurred())
-				_, err = worktree.Commit("Install setter marker", &git.CommitOptions{
-					Author: &object.Signature{
-						Name:  "Testbot",
-						Email: "[email protected]",
-						When:  time.Now(),
-					},
-				})
-				Expect(err).ToNot(HaveOccurred())
-				Expect(repo.Push(&git.PushOptions{RemoteName: "origin"})).To(Succeed())
 
 				// pull the head commit we just pushed, so it's not
 				// considered a new commit when checking for a commit
@@ -226,6 +210,7 @@ var _ = Describe("ImageUpdateAutomation", func() {
 						Namespace: updateKey.Namespace,
 					},
 					Spec: imagev1.ImageUpdateAutomationSpec{
+						RunInterval: &metav1.Duration{Duration: 2 * time.Hour}, // this is to ensure any subsequent run should be outside the scope of the testing
 						Checkout: imagev1.GitCheckoutSpec{
 							GitRepositoryRef: corev1.LocalObjectReference{
 								Name: gitRepoKey.Name,
@@ -256,22 +241,87 @@ var _ = Describe("ImageUpdateAutomation", func() {
 				Expect(err).ToNot(HaveOccurred())
 				Expect(commit.Message).To(Equal(commitMessage))
 
-				tmp, err := ioutil.TempDir("", "gotest-imageauto")
-				Expect(err).ToNot(HaveOccurred())
-				defer os.RemoveAll(tmp)
+				compareRepoWithExpected(repoURL, "testdata/appconfig-setters-expected", func(tmp string) {
+					replaceMarker(tmp, policyKey)
+				})
+			})
 
-				expected, err := ioutil.TempDir("", "gotest-imageauto-expected")
-				Expect(err).ToNot(HaveOccurred())
-				defer os.RemoveAll(expected)
-				copy.Copy("testdata/appconfig-setters-expected", expected)
-				replaceMarker(expected, policyKey)
+			It("stops updating when suspended", func() {
+				// suspend it, and check that it is marked as unready.
+				var updatePatch imagev1.ImageUpdateAutomation
+				updatePatch.Name = updateKey.Name
+				updatePatch.Namespace = updateKey.Namespace
+				updatePatch.Spec.Suspend = true
+				Expect(k8sClient.Patch(context.Background(), &updatePatch, client.Merge)).To(Succeed())
+				// wait for the suspension to reach the cache
+				var newUpdate imagev1.ImageUpdateAutomation
+				Eventually(func() bool {
+					if err := imageAutoReconciler.Get(context.Background(), updateKey, &newUpdate); err != nil {
+						return false
+					}
+					return newUpdate.Spec.Suspend
+				}, timeout, time.Second).Should(BeTrue())
+				// run the reconciliation explicitly, and make sure it
+				// doesn't do anything
+				result, err := imageAutoReconciler.Reconcile(ctrl.Request{
+					NamespacedName: updateKey,
+				})
+				Expect(err).To(BeNil())
+				Expect(result).To(Equal(ctrl.Result{})) // this ought to fail, since it should be rescheduled; but if not, additional checks lie below
+
+				var checkUpdate imagev1.ImageUpdateAutomation
+				Expect(k8sClient.Get(context.Background(), updateKey, &checkUpdate)).To(Succeed())
+				Expect(checkUpdate.Status.ObservedGeneration).NotTo(Equal(checkUpdate.ObjectMeta.Generation))
+			})
 
-				_, err = git.PlainClone(tmp, false, &git.CloneOptions{
-					URL:           repoURL,
-					ReferenceName: plumbing.NewBranchReferenceName(defaultBranch),
+			It("runs when the reconcile request annotation is added", func() {
+				println("[DEBUG]", updateKey.String())
+				// the automation has run, and is not expected to run
+				// again for 2 hours. Make a commit to the git repo
+				// which needs to be undone by automation, then add
+				// the annotation and make sure it runs again.
+				Expect(k8sClient.Get(context.Background(), updateKey, updateBySetters)).To(Succeed())
+				Expect(updateBySetters.Status.LastAutomationRunTime).ToNot(BeNil())
+				lastRunTime := updateBySetters.Status.LastAutomationRunTime.Time
+
+				commitInRepo(repoURL, "Revert image update", func(tmp string) {
+					// revert the change made by copying the old version
+					// of the file back over then restoring the setter
+					// marker
+					copy.Copy("testdata/appconfig/deploy.yaml", filepath.Join(tmp, "deploy.yaml"))
+					replaceMarker(tmp, policyKey)
+				})
+				// check that it was reverted correctly
+				compareRepoWithExpected(repoURL, "testdata/appconfig", func(tmp string) {
+					replaceMarker(tmp, policyKey)
+				})
+
+				ts := time.Now().String()
+				var updatePatch imagev1.ImageUpdateAutomation
+				updatePatch.Name = updateKey.Name
+				updatePatch.Namespace = updateKey.Namespace
+				updatePatch.ObjectMeta.Annotations = map[string]string{
+					meta.ReconcileRequestAnnotation: ts,
+				}
+				Expect(k8sClient.Patch(context.Background(), &updatePatch, client.Merge)).To(Succeed())
+
+				// ... this is where the reconciler is supposed to do its work ...
+
+				var newUpdate imagev1.ImageUpdateAutomation
+				Eventually(func() bool {
+					if err := k8sClient.Get(context.Background(), updateKey, &newUpdate); err != nil {
+						return false
+					}
+					newLastRun := newUpdate.Status.LastAutomationRunTime
+					return newLastRun != nil && newLastRun.Time.After(lastRunTime)
+				}, timeout, time.Second).Should(BeTrue())
+				// check that the annotation was recorded as seen
+				Expect(newUpdate.Status.LastHandledReconcileAt).To(Equal(ts))
+
+				// check that a new commit was made
+				compareRepoWithExpected(repoURL, "testdata/appconfig-setters-expected", func(tmp string) {
+					replaceMarker(tmp, policyKey)
 				})
-				Expect(err).ToNot(HaveOccurred())
-				test.ExpectMatchingDirectories(tmp, expected)
 			})
 		})
 	})
@@ -307,6 +357,51 @@ func waitForNewHead(repo *git.Repository) {
 	}, timeout, time.Second).Should(BeTrue())
 }
 
+func compareRepoWithExpected(repoURL, fixture string, changeFixture func(tmp string)) {
+	expected, err := ioutil.TempDir("", "gotest-imageauto-expected")
+	Expect(err).ToNot(HaveOccurred())
+	defer os.RemoveAll(expected)
+	copy.Copy(fixture, expected)
+	changeFixture(expected)
+
+	tmp, err := ioutil.TempDir("", "gotest-imageauto")
+	Expect(err).ToNot(HaveOccurred())
+	defer os.RemoveAll(tmp)
+	_, err = git.PlainClone(tmp, false, &git.CloneOptions{
+		URL:           repoURL,
+		ReferenceName: plumbing.NewBranchReferenceName(defaultBranch),
+	})
+	Expect(err).ToNot(HaveOccurred())
+	test.ExpectMatchingDirectories(tmp, expected)
+}
+
+func commitInRepo(repoURL, msg string, changeFiles func(path string)) {
+	tmp, err := ioutil.TempDir("", "gotest-imageauto")
+	Expect(err).ToNot(HaveOccurred())
+	defer os.RemoveAll(tmp)
+	repo, err := git.PlainClone(tmp, false, &git.CloneOptions{
+		URL:           repoURL,
+		ReferenceName: plumbing.NewBranchReferenceName(defaultBranch),
+	})
+	Expect(err).ToNot(HaveOccurred())
+
+	changeFiles(tmp)
+
+	worktree, err := repo.Worktree()
+	Expect(err).ToNot(HaveOccurred())
+	_, err = worktree.Add(".")
+	Expect(err).ToNot(HaveOccurred())
+	_, err = worktree.Commit(msg, &git.CommitOptions{
+		Author: &object.Signature{
+			Name:  "Testbot",
+			Email: "[email protected]",
+			When:  time.Now(),
+		},
+	})
+	Expect(err).ToNot(HaveOccurred())
+	Expect(repo.Push(&git.PushOptions{RemoteName: "origin"})).To(Succeed())
+}
+
 // Initialise a git server with a repo including the files in dir.
 func initGitRepo(gitServer *gittestserver.GitServer, fixture, repositoryPath string) error {
 	fs := memfs.New()