Merge branch 'skyler/ref-reconcile' into 'main'

Reconcile Controller On NodeSet Update

Closes #123

See merge request SchedMD/slinky-dev/slurm-operator!235

Author: SkylerMalinowski

Makefile

@@ -297,7 +297,7 @@ golangci-lint: golangci-lint-bin ## Run golangci-lint.
 golangci-lint-fmt: golangci-lint-bin ## Run golangci-lint fmt.
 	$(GOLANGCI_LINT) fmt
 
-CODECOV_PERCENT ?= 67
+CODECOV_PERCENT ?= 66
 
 .PHONY: test
 test: envtest ## Run tests.

README.md

@@ -20,6 +20,7 @@ Run [Slurm] on [Kubernetes], by [SchedMD]. A [Slinky] project.
   - [Overview](#overview)
     - [Slurm Cluster](#slurm-cluster)
   - [Features](#features)
+    - [Controller](#controller)
     - [NodeSets](#nodesets)
     - [LoginSets](#loginsets)
     - [Hybrid Support](#hybrid-support)
@@ -68,6 +69,21 @@ For additional information about Slurm, see the [slurm][slurm-docs] docs.
 
 ## Features
 
+### Controller
+
+The Slurm control-plane is responsible for scheduling Slurm workload onto its
+worker nodes and managing their states.
+
+Changes to the Slurm configuration files are automatically detected and the
+Slurm cluster is reconfigured seamlessly with zero downtime of the Slurm
+control-plane.
+
+> [!NOTE]
+> The kubelet's `configMapAndSecretChangeDetectionStrategy` and `syncFrequency`
+> settings directly affect when pods have their mounted ConfigMaps and Secrets
+> updated. By default, the kubelet is in `Watch` mode with a polling frequency
+> of 60 seconds.
+
 ### NodeSets
 
 A set of homogeneous Slurm workers (compute nodes), which are delegated to

docs/index.rst

@@ -27,6 +27,7 @@ Table of Contents
 
   - `Features <#features>`__
 
+    - `Controller <#controller>`__
     - `NodeSets <#nodesets>`__
     - `LoginSets <#loginsets>`__
     - `Hybrid Support <#hybrid-support>`__
@@ -98,6 +99,21 @@ For additional information about Slurm, see the
 Features
 --------
 
+Controller
+~~~~~~~~~~
+
+The Slurm control-plane is responsible for scheduling Slurm workload
+onto its worker nodes and managing their states.
+
+Changes to the Slurm configuration files are automatically detected and
+the Slurm cluster is reconfigured seamlessly with zero downtime of the
+Slurm control-plane.
+
+   [!NOTE] The kubelet’s ``configMapAndSecretChangeDetectionStrategy``
+   and ``syncFrequency`` settings directly affect when pods have their
+   mounted ConfigMaps and Secrets updated. By default, the kubelet is in
+   ``Watch`` mode with a polling frequency of 60 seconds.
+
 NodeSets
 ~~~~~~~~
 

internal/controller/accounting/accounting_controller.go

@@ -23,6 +23,7 @@ import (
 
 	slinkyv1beta1 "github.com/SlinkyProject/slurm-operator/api/v1beta1"
 	"github.com/SlinkyProject/slurm-operator/internal/builder"
+	"github.com/SlinkyProject/slurm-operator/internal/controller/accounting/eventhandler"
 	"github.com/SlinkyProject/slurm-operator/internal/utils/durationstore"
 	"github.com/SlinkyProject/slurm-operator/internal/utils/refresolver"
 )
@@ -111,13 +112,8 @@ func (r *AccountingReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&corev1.Service{}).
 		Owns(&corev1.ConfigMap{}).
 		Owns(&corev1.Secret{}).
-		Watches(&slinkyv1beta1.Accounting{}, &accountingEventHandler{
-			Reader:      r.Client,
-			refResolver: r.refResolver,
-		}).
-		Watches(&corev1.Secret{}, &secretEventHandler{
-			Reader: r.Client,
-		}).
+		Watches(&slinkyv1beta1.Accounting{}, eventhandler.NewAccountingEventHandler(r.Client)).
+		Watches(&corev1.Secret{}, eventhandler.NewSecretEventHandler(r.Client)).
 		WithOptions(controller.Options{
 			MaxConcurrentReconciles: maxConcurrentReconciles,
 		}).

internal/controller/accounting/accounting_controller_test.go

@@ -49,21 +49,21 @@ var _ = Describe("Accounting controller", func() {
 			accountingKey := client.ObjectKeyFromObject(accounting)
 			Eventually(func(g Gomega) {
 				g.Expect(k8sClient.Get(ctx, accountingKey, createdAccounting)).To(Succeed())
-			}).Should(Succeed())
+			}, testutils.Timeout, testutils.Internal).Should(Succeed())
 
 			By("Expecting Accounting CR Service")
 			serviceKey := accounting.ServiceKey()
 			service := &corev1.Service{}
 			Eventually(func(g Gomega) {
 				g.Expect(k8sClient.Get(ctx, serviceKey, service)).To(Succeed())
-			}).Should(Succeed())
+			}, testutils.Timeout, testutils.Internal).Should(Succeed())
 
 			By("Expecting Accounting CR Statefulset")
 			statefulsetKey := accounting.Key()
 			statefulset := &appsv1.StatefulSet{}
 			Eventually(func(g Gomega) {
 				g.Expect(k8sClient.Get(ctx, statefulsetKey, statefulset)).To(Succeed())
-			}).Should(Succeed())
+			}, testutils.Timeout, testutils.Internal).Should(Succeed())
 		}, SpecTimeout(testutils.Timeout))
 	})
 })

internal/controller/accounting/eventhandler/eventhandler_accounting.go

@@ -0,0 +1,88 @@
+// SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.
+// SPDX-License-Identifier: Apache-2.0
+
+package eventhandler
+
+import (
+	"context"
+
+	"k8s.io/client-go/util/workqueue"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	slinkyv1beta1 "github.com/SlinkyProject/slurm-operator/api/v1beta1"
+	"github.com/SlinkyProject/slurm-operator/internal/utils/objectutils"
+	"github.com/SlinkyProject/slurm-operator/internal/utils/refresolver"
+)
+
+func NewAccountingEventHandler(reader client.Reader) *AccountingEventHandler {
+	return &AccountingEventHandler{
+		Reader:      reader,
+		refResolver: refresolver.New(reader),
+	}
+}
+
+var _ handler.EventHandler = &AccountingEventHandler{}
+
+type AccountingEventHandler struct {
+	client.Reader
+	refResolver *refresolver.RefResolver
+}
+
+func (e *AccountingEventHandler) Create(
+	ctx context.Context,
+	evt event.CreateEvent,
+	q workqueue.TypedRateLimitingInterface[reconcile.Request],
+) {
+	e.enqueueRequest(ctx, evt.Object, q)
+}
+
+func (e *AccountingEventHandler) Update(
+	ctx context.Context,
+	evt event.UpdateEvent,
+	q workqueue.TypedRateLimitingInterface[reconcile.Request],
+) {
+	e.enqueueRequest(ctx, evt.ObjectNew, q)
+}
+
+func (e *AccountingEventHandler) Delete(
+	ctx context.Context,
+	evt event.DeleteEvent,
+	q workqueue.TypedRateLimitingInterface[reconcile.Request],
+) {
+	e.enqueueRequest(ctx, evt.Object, q)
+}
+
+func (e *AccountingEventHandler) Generic(
+	ctx context.Context,
+	evt event.GenericEvent,
+	q workqueue.TypedRateLimitingInterface[reconcile.Request],
+) {
+	// Intentionally blank
+}
+
+func (e *AccountingEventHandler) enqueueRequest(
+	ctx context.Context,
+	obj client.Object,
+	q workqueue.TypedRateLimitingInterface[reconcile.Request],
+) {
+	logger := log.FromContext(ctx)
+
+	accounting, ok := obj.(*slinkyv1beta1.Accounting)
+	if !ok {
+		return
+	}
+
+	list, err := e.refResolver.GetControllersForAccounting(ctx, accounting)
+	if err != nil {
+		logger.Error(err, "failed to list Controllers referencing Accounting")
+		return
+	}
+
+	for _, item := range list.Items {
+		objectutils.EnqueueRequest(q, &item)
+	}
+}

…counting/accounting_eventhandler_test.go …thandler/eventhandler_accounting_test.gointernal/controller/accounting/accounting_eventhandler_test.go renamed to internal/controller/accounting/eventhandler/eventhandler_accounting_test.go internal/controller/accounting/accounting_eventhandler_test.go renamed to internal/controller/accounting/eventhandler/eventhandler_accounting_test.go

@@ -1,14 +1,13 @@
 // SPDX-FileCopyrightText: Copyright (C) SchedMD LLC.
 // SPDX-License-Identifier: Apache-2.0
 
-package accounting
+package eventhandler
 
 import (
 	"context"
 	"testing"
 
 	slinkyv1beta1 "github.com/SlinkyProject/slurm-operator/api/v1beta1"
-	"github.com/SlinkyProject/slurm-operator/internal/utils/refresolver"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/util/workqueue"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -17,18 +16,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
-func newHandler(c client.Client) *accountingEventHandler {
-	return &accountingEventHandler{
-		Reader:      c,
-		refResolver: refresolver.New(c),
-	}
-}
-
-func newQueue() workqueue.TypedRateLimitingInterface[reconcile.Request] {
-	return workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[reconcile.Request]())
-}
-
-func Test_controllerEventHandler_Create(t *testing.T) {
+func Test_AccountingEventHandler_Create(t *testing.T) {
 	type fields struct {
 		client client.Client
 	}
@@ -87,7 +75,7 @@ func Test_controllerEventHandler_Create(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			e := newHandler(tt.fields.client)
+			e := NewAccountingEventHandler(tt.fields.client)
 			e.Create(tt.args.ctx, tt.args.evt, tt.args.q)
 			if got := tt.args.q.Len(); got > tt.want {
 				t.Errorf("Create() = %v, want %v", got, tt.want)
@@ -96,7 +84,7 @@ func Test_controllerEventHandler_Create(t *testing.T) {
 	}
 }
 
-func Test_controllerEventHandler_Update(t *testing.T) {
+func Test_AccountingEventHandler_Update(t *testing.T) {
 	type fields struct {
 		client client.Client
 	}
@@ -160,7 +148,7 @@ func Test_controllerEventHandler_Update(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			e := newHandler(tt.fields.client)
+			e := NewAccountingEventHandler(tt.fields.client)
 			e.Update(tt.args.ctx, tt.args.evt, tt.args.q)
 			if got := tt.args.q.Len(); got > tt.want {
 				t.Errorf("Create() = %v, want %v", got, tt.want)
@@ -169,7 +157,7 @@ func Test_controllerEventHandler_Update(t *testing.T) {
 	}
 }
 
-func Test_controllerEventHandler_Delete(t *testing.T) {
+func Test_AccountingEventHandler_Delete(t *testing.T) {
 	type fields struct {
 		client client.Client
 	}
@@ -228,7 +216,7 @@ func Test_controllerEventHandler_Delete(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			e := newHandler(tt.fields.client)
+			e := NewAccountingEventHandler(tt.fields.client)
 			e.Delete(tt.args.ctx, tt.args.evt, tt.args.q)
 			if got := tt.args.q.Len(); got > tt.want {
 				t.Errorf("Create() = %v, want %v", got, tt.want)
@@ -237,7 +225,7 @@ func Test_controllerEventHandler_Delete(t *testing.T) {
 	}
 }
 
-func Test_controllerEventHandler_Generic(t *testing.T) {
+func Test_AccountingEventHandler_Generic(t *testing.T) {
 	type fields struct {
 		client client.Client
 	}
@@ -267,7 +255,7 @@ func Test_controllerEventHandler_Generic(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			e := newHandler(tt.fields.client)
+			e := NewAccountingEventHandler(tt.fields.client)
 			e.Generic(tt.args.ctx, tt.args.evt, tt.args.q)
 			if got := tt.args.q.Len(); got > tt.want {
 				t.Errorf("Create() = %v, want %v", got, tt.want)