observe sts #1

Author: iaoiui

app/main.go

@@ -101,19 +101,42 @@ func listPod(clientset *kubernetes.Clientset) {
 		msg := generateAlertMsg(notReadyPods)
 		sendAlertToTeams("Pod Defect Alert", msg, teamsEndpoint)
 	}
-	// Issue #1
+
 	// https://github.com/iaoiui/PodMonitoringTool/issues/1
 	observeReplicaNumbers(clientset)
 
 }
 
+// observe replica number of deployment and statefulsets
 func observeReplicaNumbers(clientset *kubernetes.Clientset) {
 	// get not ready deployment
 	notReadyDeployments := getNotReadyDeployments(clientset)
 	msg := generateAlertMsgForDeployment(notReadyDeployments)
 	sendAlertToTeams("Deployment Defect Alert", msg, teamsEndpoint)
 
-	// TODO get not ready statefulsets
+	// get not ready statefulsets
+	notReadyStatefulsets := getNotReadyStatefulsets(clientset)
+	msg = generateAlertMsgForStatefulset(notReadyStatefulsets)
+	sendAlertToTeams("Statefulsets Defect Alert", msg, teamsEndpoint)
+}
+
+func getNotReadyStatefulsets(clientset *kubernetes.Clientset) []appsv1.StatefulSet {
+	statefulsets, err := clientset.AppsV1().StatefulSets(namespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		panic(err.Error())
+	}
+	notReadyStatefulsets := []appsv1.StatefulSet{}
+
+	for _, s := range statefulsets.Items {
+		desiredReplicas := s.Status.Replicas
+		availableReplicas := s.Status.ReadyReplicas
+		if desiredReplicas != availableReplicas {
+			notReadyStatefulsets = append(notReadyStatefulsets, s)
+		}
+
+	}
+
+	return notReadyStatefulsets
 }
 
 func getNotReadyDeployments(clientset *kubernetes.Clientset) []appsv1.Deployment {
@@ -135,6 +158,18 @@ func getNotReadyDeployments(clientset *kubernetes.Clientset) []appsv1.Deployment
 	return notReadyDeployments
 }
 
+func generateAlertMsgForStatefulset(statefulsets []appsv1.StatefulSet) string {
+	msg := ""
+	log.Printf("%v statefulsets is not ready \n", len(statefulsets))
+	msg += fmt.Sprintf("# **%v statefulset is not ready** \n", len(statefulsets)) + "\n"
+	for i, sts := range statefulsets {
+		log.Println(i + 1)
+		log.Println("\t", sts.Name, "\t")
+		msg += fmt.Sprintln("\t Namespace: \t", sts.Namespace, "StatefulSet: \t", sts.Name, ", availableReplicas: \t", sts.Status.ReadyReplicas, ", desiredReplicas: \t", sts.Status.Replicas) + "\n"
+	}
+	return msg
+}
+
 func generateAlertMsgForDeployment(deployments []appsv1.Deployment) string {
 	msg := ""
 	log.Printf("%v deployments is not ready \n", len(deployments))

kubernetes/failure_test/test_sts.yaml

@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: stateful
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: stateful
+  serviceName: stateful
+  template:
+    metadata:
+      labels:
+        app: stateful
+    spec:
+      containers:
+      - image: busybox
+        name: stateful
+        command: ["sh", "-c"]
+        args:
+        - |
+          touch /data/hostname
+          while true; do
+            echo "`date`: `hostname`" >> /data/hostname
+            sleep 1
+          done
+          touch /data/ready
+        volumeMounts:
+        - name: data
+          mountPath: /data
+        readinessProbe:
+          exec:
+            command:
+            - cat
+            - /data/ready
+          initialDelaySeconds: 5
+          periodSeconds: 5
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      resources:
+        requests:
+          storage: 10Mi