can work inside cluster

Author: iaoiui

Dockerfile

@@ -0,0 +1,8 @@
+FROM golang:latest
+
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY ./app/main.go .
+RUN go build -o main .
+CMD ["./main"]

README.md

@@ -19,6 +19,60 @@ Once this tool detect pod failure, this tool send alert to MS Teams.
 
 # Usage
 
+**This tool works as both inside cluster pod and go application outside cluster.**
+
+# Inside Cluster
+
+## Configuration
+
+**Set environment variables like below**
+
+Please modify kubernetes/deploy_pod_monitoring_tool.yaml like below.
+
+```
+        env:
+        # !! replace TEAMS_ENDPOINT with your Teams webhook endpoint
+        # - name: TEAMS_ENDPOINT
+        #   value: "https://outlook.office.com/webhook/XXXXXXXX"
+        - name: OBSERVE_PERIOD
+          value: "10"
+        - name: OBSERVED_NAMESPACE
+          value: "default"
+```
+
+## Launch
+
+**Run this tool simply**
+
+```
+kubectl apply -f ./kubernetes/prerequisite.yaml
+kubectl apply -f ./kubernetes/deploy_pod_monitoring_tool.yaml
+```
+
+```
+❯ kubectl get po -n pod-monitoring
+NAME                                   READY   STATUS        RESTARTS   AGE
+pod-monitoring-tool-555688b96-b2z2t    1/1     Running       0          4s
+```
+
+## Verification
+
+**The following manifests can be used to verify that this tool works successfully.**
+
+```
+❯ kubectl apply -f ./kubernetes/failure_test
+deployment.apps/nginx created
+deployment.apps/nginx2 created
+```
+
+**And then, you can find 2 pods is in failed state.**
+
+The alert can be found in your MS Teams like below.
+
+![](./assets/teams_alert.PNG)
+
+# Outside Cluster
+
 ## Configuration
 
 **Set environment variables like below**
@@ -45,7 +99,7 @@ OBSERVE_PERIOD=10
 **The following manifests can be used to verify that this tool works successfully.**
 
 ```
-❯ kubectl apply -f .
+❯ kubectl apply -f ./kubernetes/failure_test
 deployment.apps/nginx created
 deployment.apps/nginx2 created
 ```
@@ -64,4 +118,9 @@ deployment.apps/nginx2 created
 
 The alert can be found in your MS Teams like below.
 
-![](teams_alert.PNG)
+![](./assets/teams_alert.PNG)
+
+# Options
+
+if you set TEAMS_HEARTBEAT_ENDPOINT variable, succesful notification can be notified every observation period like below.
+![](./assets/heartbeat.png)

main.go app/main.gomain.go renamed to app/main.go

@@ -20,7 +20,7 @@ import (
 )
 
 // namespace which is observed
-var namespace string = getEnv("NAMESPACE", "default")
+var namespace string = getEnv("OBSERVED_NAMESPACE", "default")
 
 // observe period (sec)
 var observePeriod = 10
@@ -30,6 +30,9 @@ var observePeriod = 10
 // TODO Erace Specific endpoint
 var teamsEndpoint string = getEnv("TEAMS_ENDPOINT", "")
 
+// (Optional) teamsHeartbeatEndpoint is a endpoint where this tool alert when all pod works successfully
+var teamsHeartbeatEndpoint string = getEnv("TEAMS_HEARTBEAT_ENDPOINT", "")
+
 func getEnv(key, fallback string) string {
 	if value, ok := os.LookupEnv(key); ok {
 		return value
@@ -43,6 +46,12 @@ func main() {
 		log.Fatal("please set TEAMS_ENDPOINT\n")
 		return
 	}
+
+	if teamsHeartbeatEndpoint != "" {
+		// send heartbeat to teams at every observation period
+		log.Printf("Heartbeat will be notified every %v seconds\n", observePeriod)
+	}
+
 	observePeriod, err := strconv.Atoi(getEnv("OBSERVE_PERIOD", string(10)))
 	if err != nil {
 		log.Fatal(err)
@@ -81,9 +90,14 @@ func listPod(clientset *kubernetes.Clientset) {
 	// There is no NotReady Pods
 	if len(notReadyPods) == 0 {
 		log.Println("All Pod work succesfully")
+		if teamsHeartbeatEndpoint != "" {
+			// send heartbeat to teams at every observation period
+			sendAlertToTeams("PodMonitoringTool Heartbeat", "All Pod work succesfully", teamsHeartbeatEndpoint)
+		}
+
 	} else {
 		msg := generateAlertMsg(notReadyPods)
-		sendAlertToTeams(msg)
+		sendAlertToTeams("Pod Defect Alert", msg, teamsEndpoint)
 	}
 
 }
@@ -128,26 +142,41 @@ func hasNotReadyContainer(p v1.Pod) bool {
 }
 
 func getKubeConfig() *rest.Config {
+	// Inside Cluster Config
+	if exists("/run/secrets/kubernetes.io/serviceaccount") {
+		config, err := rest.InClusterConfig()
+		if err != nil {
+			panic(err.Error())
+		}
+		return config
+	}
+	// Outside Cluster Config
 	var kubeconfig *string
 	if home := homeDir(); home != "" {
 		kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
 	} else {
 		kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
 	}
+
 	flag.Parse()
 
 	// use the current context in kubeconfig
 	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
 	if err != nil {
 		panic(err.Error())
 	}
+
 	return config
 }
+func exists(name string) bool {
+	_, err := os.Stat(name)
+	return !os.IsNotExist(err)
+}
 
-func sendAlertToTeams(msg string) {
-	b := fmt.Sprintf(`{ "title": "Pod Defect Alert", "text": "%v"}`, msg)
+func sendAlertToTeams(title, msg, endpoint string) {
+	b := fmt.Sprintf(`{ "title": "%v", "text": "%v"}`, title, msg)
 	body := strings.NewReader(b)
-	req, err := http.NewRequest("POST", teamsEndpoint, body)
+	req, err := http.NewRequest("POST", endpoint, body)
 	if err != nil {
 		log.Println(err)
 		return

assets/heartbeat.png

@@ -0,0 +1,2 @@
+docker build . -t pod_monitoring_tool 
+kind load docker-image pod_monitoring_tool:latest

teams_alert.png assets/teams_alert.pngteams_alert.png renamed to assets/teams_alert.png

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: pod-monitoring-tool
+  name: pod-monitoring-tool
+  namespace: pod-monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: pod-monitoring-tool
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        app: pod-monitoring-tool
+    spec:
+      serviceAccountName: pod-monitoring
+      serviceAccount: pod-monitoring
+      containers:
+      - image: pod_monitoring_tool
+        name: pod-monitoring-tool
+        imagePullPolicy: Never
+        env: 
+        # !! replace TEAMS_ENDPOINT with your Teams webhook endpoint
+        # - name: TEAMS_ENDPOINT
+        #   value: "https://outlook.office.com/webhook/XXXXXXXX"
+        # - name: TEAMS_HEARTBEAT_ENDPOINT
+        #   value: "https://outlook.office.com/webhook/XXXXXXXX"
+        - name: OBSERVE_PERIOD
+          value: "10"
+        - name: OBSERVED_NAMESPACE
+          value: "default"
+        resources: 
+          requests:
+            memory: "64Mi"
+            cpu: 10m
+              

dev/build.sh

@@ -0,0 +1,24 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: pod-monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: pod-monitoring
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: pod-monitoring
+  namespace: pod-monitoring
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: pod-monitoring
+  namespace: pod-monitoring