Merge pull request #588 from brownleej/buggify-crash-loop

Add an option to put pods in a crash loop state for testing purposes

Author: brownleej

api/v1beta1/foundationdbcluster_types.go

@@ -2427,6 +2427,10 @@ type RequiredAddressSet struct {
 type BuggifyConfig struct {
 	// NoSchedule defines a list of instance IDs that should fail to schedule.
 	NoSchedule []string `json:"noSchedule,omitempty"`
+
+	// CrashLoops defines a list of instance IDs that should be put into a
+	// crash looping state.
+	CrashLoop []string `json:"crashLoop,omitempty"`
 }
 
 // FdbVersion represents a version of FoundationDB.

api/v1beta1/zz_generated.deepcopy.go

@@ -68,6 +68,10 @@ spec:
                   type: boolean
                 buggify:
                   properties:
+                    crashLoop:
+                      items:
+                        type: string
+                      type: array
                     noSchedule:
                       items:
                         type: string

config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml

@@ -30,13 +30,19 @@ spec:
           - id: dc1
             priority: 1
           - id: dc3
+            satellite: 1
+            priority: 2
+          - id: dc2
             satellite: 1
             priority: 1
         satellite_logs: 3
       - datacenters:
           - id: dc2
             priority: 0
           - id: dc3
+            satellite: 1
+            priority: 2
+          - id: dc1
             satellite: 1
             priority: 1
         satellite_logs: 3

config/samples/multi_dc/final.yaml

@@ -27,7 +27,7 @@ spec:
     regions:
       - datacenters:
           - id: $dc
-        priority: 1
+            priority: 1
   processes:
     general:
       volumeClaimTemplate:

config/samples/multi_dc/stage_1.yaml

@@ -138,6 +138,7 @@ func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request c
 		CheckClientCompatibility{},
 		ReplaceMisconfiguredPods{},
 		ReplaceFailedPods{},
+		DeletePodsForBuggification{},
 		AddProcessGroups{},
 		AddServices{},
 		AddPVCs{},
@@ -839,7 +840,7 @@ type PodLifecycleManager interface {
 	CanDeletePods(*FoundationDBClusterReconciler, ctx.Context, *fdbtypes.FoundationDBCluster) (bool, error)
 
 	// UpdatePods updates a list of pods to match the latest specs.
-	UpdatePods(*FoundationDBClusterReconciler, ctx.Context, *fdbtypes.FoundationDBCluster, []FdbInstance) error
+	UpdatePods(reconciler *FoundationDBClusterReconciler, context ctx.Context, cluster *fdbtypes.FoundationDBCluster, instances []FdbInstance, unsafe bool) error
 
 	// UpdateImageVersion updates a container's image.
 	UpdateImageVersion(*FoundationDBClusterReconciler, ctx.Context, *fdbtypes.FoundationDBCluster, FdbInstance, int, string) error
@@ -969,7 +970,7 @@ func (manager StandardPodLifecycleManager) CanDeletePods(r *FoundationDBClusterR
 }
 
 // UpdatePods updates a list of pods to match the latest specs.
-func (manager StandardPodLifecycleManager) UpdatePods(r *FoundationDBClusterReconciler, context ctx.Context, cluster *fdbtypes.FoundationDBCluster, instances []FdbInstance) error {
+func (manager StandardPodLifecycleManager) UpdatePods(r *FoundationDBClusterReconciler, context ctx.Context, cluster *fdbtypes.FoundationDBCluster, instances []FdbInstance, unsafe bool) error {
 	for _, instance := range instances {
 		err := r.Delete(context, instance.Pod)
 		if err != nil {

controllers/cluster_controller.go

@@ -257,6 +257,28 @@ var _ = Describe(string(fdbtypes.ProcessClassClusterController), func() {
 			})
 		})
 
+		Context("when buggifying a pod to make it crash loop", func() {
+			BeforeEach(func() {
+				cluster.Spec.Buggify.CrashLoop = []string{"storage-1"}
+				err = k8sClient.Update(context.TODO(), cluster)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("should add the crash loop flag", func() {
+				pods := &corev1.PodList{}
+				err = k8sClient.List(context.TODO(), pods, getListOptions(cluster)...)
+				Expect(len(pods.Items)).To(Equal(len(originalPods.Items)))
+				sortPodsByID(pods)
+
+				pod := pods.Items[firstStorageIndex]
+				Expect(pod.ObjectMeta.Labels[FDBInstanceIDLabel]).To(Equal("storage-1"))
+
+				mainContainer := pod.Spec.Containers[0]
+				Expect(mainContainer.Name).To(Equal("foundationdb"))
+				Expect(mainContainer.Args).To(Equal([]string{"crash-loop"}))
+			})
+		})
+
 		Context("with a decreased process count", func() {
 			BeforeEach(func() {
 				cluster.Spec.ProcessCounts.Storage = 3

controllers/cluster_controller_test.go

@@ -0,0 +1,104 @@
+/*
+ * delete_pods_for_buggification.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+	ctx "context"
+	"time"
+
+	fdbtypes "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta1"
+)
+
+// DeletePodsForBuggification provides a reconciliation step for recreating
+// pods with new pod specs when buggifying the config.
+type DeletePodsForBuggification struct{}
+
+// Reconcile runs the reconciler's work.
+func (d DeletePodsForBuggification) Reconcile(r *FoundationDBClusterReconciler, context ctx.Context, cluster *fdbtypes.FoundationDBCluster) (bool, error) {
+	instances, err := r.PodLifecycleManager.GetInstances(r, cluster, context, getPodListOptions(cluster, "", "")...)
+	if err != nil {
+		return false, err
+	}
+
+	updates := make([]FdbInstance, 0)
+
+	removals := make(map[string]bool)
+	for _, processGroup := range cluster.Status.ProcessGroups {
+		if processGroup.Remove {
+			removals[processGroup.ProcessGroupID] = true
+		}
+	}
+
+	crashLoopPods := make(map[string]bool, len(cluster.Spec.Buggify.CrashLoop))
+	crashLoopAll := false
+	for _, instanceID := range cluster.Spec.Buggify.CrashLoop {
+		if instanceID == "*" {
+			crashLoopAll = true
+		} else {
+			crashLoopPods[instanceID] = true
+		}
+	}
+
+	for _, instance := range instances {
+		if instance.Pod == nil {
+			continue
+		}
+
+		instanceID := instance.GetInstanceID()
+		_, pendingRemoval := removals[instanceID]
+		if pendingRemoval {
+			continue
+		}
+
+		inCrashLoop := false
+		for _, container := range instance.Pod.Spec.Containers {
+			if container.Name == "foundationdb" && len(container.Args) > 0 {
+				inCrashLoop = container.Args[0] == "crash-loop"
+			}
+		}
+
+		shouldCrashLoop := crashLoopAll || crashLoopPods[instanceID]
+
+		if shouldCrashLoop != inCrashLoop {
+			log.Info("Deleting pod for buggification", "instanceID", instanceID, "shouldCrashLoop", shouldCrashLoop, "inCrashLoop", inCrashLoop)
+			updates = append(updates, instance)
+		}
+	}
+
+	if len(updates) > 0 {
+		log.Info("Deleting pods", "namespace", cluster.Namespace, "cluster", cluster.Name, "count", len(updates))
+		r.Recorder.Event(cluster, "Normal", "UpdatingPods", "Recreating pods for buggification")
+
+		err = r.PodLifecycleManager.UpdatePods(r, context, cluster, updates, true)
+		if err != nil {
+			return false, err
+		}
+
+		return false, nil
+	}
+	return true, nil
+}
+
+// RequeueAfter returns the delay before we should run the reconciliation
+// again.
+func (d DeletePodsForBuggification) RequeueAfter() time.Duration {
+	return 0
+}