feat: adds cleanup logic to operator (#165)

Author: jmgilman

foundry/operator/api/v1alpha1/release_types.go

@@ -27,6 +27,12 @@ type ReleaseDeploymentSpec struct {
 
 	// ReleaseID is the identifier for the release this deployment belongs to.
 	ReleaseID string `json:"release_id"`
+
+	// TTL specifies the time to live for this deployment after completion (in seconds).
+	// After this period has elapsed since completion, the operator will delete the resource.
+	// +optional
+	// +kubebuilder:default=300
+	TTL int64 `json:"ttl,omitempty"`
 }
 
 // GitSpec defines the source Git repository for the release.
@@ -49,6 +55,10 @@ type ReleaseDeploymentStatus struct {
 
 	// State is the current state of the release.
 	State string `json:"state"`
+
+	// CompletionTime represents the time when this deployment completed (succeeded or failed).
+	// +optional
+	CompletionTime *metav1.Time `json:"completionTime,omitempty"`
 }
 
 // +kubebuilder:object:root=true

foundry/operator/api/v1alpha1/zz_generated.deepcopy.go

@@ -239,7 +239,7 @@ func main() {
 	if err = (&controller.ReleaseDeploymentReconciler{
 		Client:            mgr.GetClient(),
 		Config:            cfg,
-		DeploymentHandler: handlers.NewReleaseDeploymentHandler(context.Background(), apiClient),
+		DeploymentHandler: handlers.NewReleaseDeploymentHandler(context.Background(), apiClient, mgr.GetClient()),
 		Logger:            logger,
 		ManifestStore:     deployment.NewDefaultManifestGeneratorStore(),
 		Remote:            remote.GoGitRemoteInteractor{},

foundry/operator/cmd/main.go

@@ -46,13 +46,25 @@ spec:
                 description: ReleaseID is the identifier for the release this deployment
                   belongs to.
                 type: string
+              ttl:
+                default: 300
+                description: |-
+                  TTL specifies the time to live for this deployment after completion (in seconds).
+                  After this period has elapsed since completion, the operator will delete the resource.
+                format: int64
+                type: integer
             required:
             - id
             - release_id
             type: object
           status:
             description: ReleaseDeploymentStatus defines the observed state of Release.
             properties:
+              completionTime:
+                description: CompletionTime represents the time when this deployment
+                  completed (succeeded or failed).
+                format: date-time
+                type: string
               conditions:
                 items:
                   description: Condition contains details for one aspect of the current

foundry/operator/config/crd/bases/foundry.projectcatalyst.io_releasedeployments.yaml

@@ -7,7 +7,7 @@ rules:
 - apiGroups:
   - foundry.projectcatalyst.io
   resources:
-  - releasedeployments
+  - releases
   verbs:
   - create
   - delete
@@ -19,13 +19,13 @@ rules:
 - apiGroups:
   - foundry.projectcatalyst.io
   resources:
-  - releasedeployments/finalizers
+  - releases/finalizers
   verbs:
   - update
 - apiGroups:
   - foundry.projectcatalyst.io
   resources:
-  - releasedeployments/status
+  - releases/status
   verbs:
   - get
   - patch

foundry/operator/config/rbac/role.yaml

@@ -19,6 +19,7 @@ package controller
 import (
 	"context"
 	"log/slog"
+	"time"
 
 	"cuelang.org/go/cue/cuecontext"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -48,17 +49,17 @@ type ReleaseDeploymentReconciler struct {
 	SecretStore       secrets.SecretStore
 }
 
-// +kubebuilder:rbac:groups=foundry.projectcatalyst.io,resources=releases,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=foundry.projectcatalyst.io,resources=releases/status,verbs=get;update;patch
-// +kubebuilder:rbac:groups=foundry.projectcatalyst.io,resources=releases/finalizers,verbs=update
+// +kubebuilder:rbac:groups=foundry.projectcatalyst.io,resources=releasedeployments,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=foundry.projectcatalyst.io,resources=releasedeployments/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=foundry.projectcatalyst.io,resources=releasedeployments/finalizers,verbs=update
 
 func (r *ReleaseDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	log := log.FromContext(ctx)
 
 	// 1. Fetch the ReleaseDeployment resource
 	var resource foundryv1alpha1.ReleaseDeployment
 	if err := r.Get(ctx, req.NamespacedName, &resource); err != nil {
-		log.Error(err, "unable to fetch Release")
+		log.Error(err, "unable to fetch ReleaseDeployment")
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
@@ -68,12 +69,39 @@ func (r *ReleaseDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re
 		log.Error(err, "unable to load deployment")
 		return ctrl.Result{}, err
 	}
-	release := r.DeploymentHandler.Release()
 
-	// 3. Check if the deployment has already been completed
+	// 3. Check for TTL expiration on completed deployments
 	if r.DeploymentHandler.IsCompleted() {
-		log.Info("Deployment already finished")
-		return ctrl.Result{}, nil
+		log.Info("Deployment is completed, checking TTL")
+
+		if err := r.DeploymentHandler.UpdateCompletionTime(); err != nil {
+			log.Error(err, "unable to update completion time")
+			return ctrl.Result{}, err
+		}
+
+		expired, timeUntilExpiry := r.DeploymentHandler.IsExpired()
+		if expired {
+			log.Info("Deployment TTL has expired, deleting resource",
+				"completionTime", resource.Status.CompletionTime,
+				"ttl", resource.Spec.TTL)
+
+			if err := r.Delete(ctx, &resource); err != nil {
+				log.Error(err, "unable to delete expired deployment")
+				return ctrl.Result{}, err
+			}
+
+			return ctrl.Result{}, nil
+		}
+
+		// Add a 5 second buffer to avoid requeuing too early
+		requeueAfter := timeUntilExpiry + 5*time.Second
+		log.Info("Deployment not expired yet, requeuing",
+			"completionTime", resource.Status.CompletionTime,
+			"ttl", resource.Spec.TTL,
+			"timeUntilExpiry", timeUntilExpiry.String(),
+			"requeueAfter", requeueAfter.String())
+
+		return ctrl.Result{RequeueAfter: requeueAfter}, nil
 	}
 
 	// 4. Check if max attempts have been reached
@@ -94,19 +122,15 @@ func (r *ReleaseDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re
 		return ctrl.Result{}, err
 	}
 
-	// resource.Status.State = "Deploying"
-	// if err := r.Status().Update(ctx, &resource); err != nil {
-	// 	log.Error(err, "unable to update Release status")
-	// 	return ctrl.Result{}, err
-	// }
-
 	// 6. Open repos
 	log.Info("Opening deployment repo", "url", r.Config.Deployer.Git.Url)
 	if err := r.RepoHandler.LoadDeploymentRepo(r.Config.Deployer.Git.Url, r.Config.Deployer.Git.Ref); err != nil {
 		log.Error(err, "unable to load deployment repo")
+		r.DeploymentHandler.AddErrorEvent(err, "Unable to load deployment repo")
 		return ctrl.Result{}, err
 	}
 
+	release := r.DeploymentHandler.Release()
 	log.Info("Opening source repo", "url", release.SourceRepo)
 	if err := r.RepoHandler.LoadSourceRepo(release.SourceRepo, release.SourceCommit); err != nil {
 		log.Error(err, "unable to load source repo")
@@ -160,6 +184,7 @@ func (r *ReleaseDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re
 		return ctrl.Result{}, err
 	}
 
+	// Requeue to check TTL later
 	return ctrl.Result{}, nil
 }
 

foundry/operator/internal/controller/release_controller.go

@@ -33,7 +33,7 @@ import (
 var _ = Describe("ReleaseDeployment Controller", func() {
 	Context("When reconciling a release deployment", func() {
 		const (
-			timeout  = time.Second * 10
+			timeout  = time.Second * 5
 			interval = time.Millisecond * 250
 		)
 
@@ -43,46 +43,40 @@ var _ = Describe("ReleaseDeployment Controller", func() {
 			)
 
 			BeforeAll(func() {
-				// Initialize the test environment
 				env.Init(
 					map[string]string{
 						"project/blueprint.cue": newRawBlueprint(),
 					},
 					map[string]string{
 						"root/test/project/env.cue": `main: values: { key1: "value1" }`,
 					},
+					k8sClient,
 				)
 				env.ConfigureController(controller)
 
-				release := &foundryv1alpha1.ReleaseDeployment{}
-				err := k8sClient.Get(ctx, getNamespacedName(env.releaseDeploymentObj), release)
+				err := k8sClient.Get(ctx, getNamespacedName(env.releaseDeploymentObj), env.releaseDeploymentObj)
 				if err != nil && errors.IsNotFound(err) {
 					Expect(k8sClient.Create(ctx, env.releaseDeploymentObj)).To(Succeed())
 				}
 			})
 
-			// AfterEach(func() {
-			// 	// TODO(user): Cleanup logic after each test, like removing the resource instance.
-			// 	resource := &foundryv1alpha1.Release{}
-			// 	err := k8sClient.Get(ctx, typeNamespacedName, resource)
-			// 	Expect(err).NotTo(HaveOccurred())
-
-			// 	By("Cleanup the specific resource instance Release")
-			// 	Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
-			// })
+			AfterAll(func() {
+				err := k8sClient.Get(ctx, getNamespacedName(env.releaseDeploymentObj), env.releaseDeploymentObj)
+				if err == nil {
+					Expect(k8sClient.Delete(ctx, env.releaseDeploymentObj)).To(Succeed())
+				}
+			})
 
 			It("should set the status to succeeded", func() {
-				//release := &foundryv1alpha1.ReleaseDeployment{}
 				Eventually(func(g Gomega) {
-					//g.Expect(k8sClient.Get(ctx, getNamespacedName(releaseDeploymentObj), release)).To(Succeed())
-					//g.Expect(release.Status.State).To(Equal("Deployed"))
+					g.Expect(k8sClient.Get(ctx, getNamespacedName(env.releaseDeploymentObj), env.releaseDeploymentObj)).To(Succeed())
+					g.Expect(env.releaseDeploymentObj.Status.State).To(Equal(string(api.DeploymentStatusSucceeded)))
 					g.Expect(env.releaseDeployment.Status).To(Equal(api.DeploymentStatusSucceeded))
 					g.Expect(hasEvent(env.releaseDeployment.Events, "DeploymentSucceeded", "Deployment has succeeded")).To(BeTrue())
 				}, timeout, interval).Should(Succeed())
 			})
 
 			It("should have called the API client to get the deployment", func() {
-				Expect(len(env.mockClient.GetDeploymentCalls())).To(Equal(1))
 				Expect(env.mockClient.GetDeploymentCalls()[0].DeployID).To(Equal(env.releaseDeployment.ID))
 				Expect(env.mockClient.GetDeploymentCalls()[0].ReleaseID).To(Equal(env.releaseDeployment.Release.ID))
 			})
@@ -124,6 +118,26 @@ var _ = Describe("ReleaseDeployment Controller", func() {
 				Expect(err).NotTo(HaveOccurred())
 				Expect(string(got)).To(Equal(string(env.manifestContent)))
 			})
+
+			Context("when the ttl expires", func() {
+				BeforeAll(func() {
+					err := k8sClient.Get(ctx, getNamespacedName(env.releaseDeploymentObj), env.releaseDeploymentObj)
+					Expect(err).To(Succeed())
+
+					env.mockClock.Advance(time.Hour * 1)
+					env.releaseDeploymentObj.Spec.TTL = 1
+					Expect(k8sClient.Update(ctx, env.releaseDeploymentObj)).To(Succeed())
+				})
+
+				It("should delete the deployment", func() {
+					Eventually(func(g Gomega) {
+						release := &foundryv1alpha1.ReleaseDeployment{}
+						err := k8sClient.Get(ctx, getNamespacedName(env.releaseDeploymentObj), release)
+						g.Expect(err).To(HaveOccurred())
+						g.Expect(errors.IsNotFound(err)).To(BeTrue())
+					}, timeout, interval).Should(Succeed())
+				})
+			})
 		})
 	})
 })

foundry/operator/internal/controller/release_controller_test.go

@@ -0,0 +1,32 @@
+package controller
+
+import "time"
+
+// MockClock is a Clock that can be controlled for testing.
+type MockClock struct {
+	CurrentTime time.Time
+}
+
+func (m *MockClock) Now() time.Time {
+	return m.CurrentTime
+}
+
+func (m *MockClock) Since(t time.Time) time.Duration {
+	return m.CurrentTime.Sub(t)
+}
+
+func (m *MockClock) Until(t time.Time) time.Duration {
+	return t.Sub(m.CurrentTime)
+}
+
+func (m *MockClock) SetTime(t time.Time) {
+	m.CurrentTime = t
+}
+
+func (m *MockClock) Advance(d time.Duration) {
+	m.CurrentTime = m.CurrentTime.Add(d)
+}
+
+func NewMockClock(initialTime time.Time) *MockClock {
+	return &MockClock{CurrentTime: initialTime}
+}

foundry/operator/internal/controller/suite_clock_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"time"
 
 	"cuelang.org/go/cue/cuecontext"
 	"github.com/go-git/go-billy/v5"
@@ -29,6 +30,7 @@ import (
 	"github.com/input-output-hk/catalyst-forge/lib/tools/testutils"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type mockEnv struct {
@@ -39,6 +41,7 @@ type mockEnv struct {
 	deployFs              *bfs.BillyFs
 	manifestContent       string
 	mockClient            *am.ClientMock
+	mockClock             *MockClock
 	mockDeploymentHandler *handlers.ReleaseDeploymentHandler
 	mockManifestStore     deployment.ManifestGeneratorStore
 	mockRemote            *rm.GitRemoteInteractorMock
@@ -50,7 +53,7 @@ type mockEnv struct {
 	sourceRepo            *gg.Repository
 }
 
-func (m *mockEnv) Init(sourceFiles, deployFiles map[string]string) {
+func (m *mockEnv) Init(sourceFiles, deployFiles map[string]string, k8sClient k8sclient.Client) {
 	var err error
 
 	// Setup filesystems
@@ -122,7 +125,8 @@ func (m *mockEnv) Init(sourceFiles, deployFiles map[string]string) {
 	m.mockSecretStore = tu.NewMockSecretStore(map[string]string{"token": "value"})
 
 	// Setup the mock handlers
-	m.mockDeploymentHandler = handlers.NewReleaseDeploymentHandler(context.Background(), m.mockClient)
+	m.mockClock = NewMockClock(time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC))
+	m.mockDeploymentHandler = handlers.NewReleaseDeploymentHandlerWithClock(context.Background(), m.mockClient, k8sClient, m.mockClock)
 	m.mockRepoHandler = handlers.NewRepoHandler(m.deployFs, m.sourceFs, testutils.NewNoopLogger(), m.mockRemote, "token")
 
 	// Initialize the source repository (to properly set the source commit)