monitoring PodOOMKilling event, look up Pod's name and put it into event message

Author: Ruijian Zhang

cmd/nodeproblemdetector/node_problem_detector.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"k8s.io/node-problem-detector/pkg/systemlogmonitor"
 	"os"
 
 	"github.com/golang/glog"
@@ -55,6 +56,11 @@ func main() {
 		glog.Fatalf("No problem daemon is configured")
 	}
 
+	if c := systemlogmonitor.InitK8sClientOrDie(npdo); c != nil {
+		glog.Info("System Log Monitor K8S client initialized")
+	} else {
+		glog.Error("Failed to initialize System Log Monitor K8S client")
+	}
 	// Initialize exporters.
 	defaultExporters := []types.Exporter{}
 	if ke := k8sexporter.NewExporterOrDie(npdo); ke != nil {

pkg/systemlogmonitor/log_monitor.go

@@ -19,7 +19,16 @@ package systemlogmonitor
 import (
 	"encoding/json"
 	"io/ioutil"
+	"k8s.io/heapster/common/kubernetes"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/node-problem-detector/cmd/options"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
 	"time"
+	"fmt"
 
 	"github.com/golang/glog"
 
@@ -32,9 +41,19 @@ import (
 	"k8s.io/node-problem-detector/pkg/types"
 	"k8s.io/node-problem-detector/pkg/util"
 	"k8s.io/node-problem-detector/pkg/util/tomb"
+	"k8s.io/node-problem-detector/pkg/version"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const SystemLogMonitorName = "system-log-monitor"
+const (
+	SystemLogMonitorName = "system-log-monitor"
+	OOMREASON            = "PodOOMKilling"
+)
+
+var (
+	uuidRegx                      *regexp.Regexp
+	k8sClient                     *clientset.Clientset
+)
 
 func init() {
 	problemdaemon.Register(
@@ -44,6 +63,10 @@ func init() {
 			CmdOptionDescription:     "Set to config file paths."})
 }
 
+func init() {
+	uuidRegx = regexp.MustCompile("[0-9a-f]{8}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{12}")
+}
+
 type logMonitor struct {
 	configPath string
 	watcher    watchertypes.LogWatcher
@@ -55,6 +78,17 @@ type logMonitor struct {
 	tomb       *tomb.Tomb
 }
 
+func InitK8sClientOrDie(options *options.NodeProblemDetectorOptions) *clientset.Clientset{
+	uri, _ := url.Parse(options.ApiServerOverride)
+	cfg, err := kubernetes.GetKubeClientConfig(uri)
+	if err != nil {
+		panic(err)
+	}
+	cfg.UserAgent = fmt.Sprintf("%s/%s", filepath.Base(os.Args[0]), version.Version())
+	k8sClient = clientset.NewForConfigOrDie(cfg)
+	return k8sClient
+}
+
 // NewLogMonitorOrDie create a new LogMonitor, panic if error occurs.
 func NewLogMonitorOrDie(configPath string) types.Monitor {
 	l := &logMonitor{
@@ -167,6 +201,22 @@ func (l *logMonitor) generateStatus(logs []*logtypes.Log, rule systemlogtypes.Ru
 	// We use the timestamp of the first log line as the timestamp of the status.
 	timestamp := logs[0].Timestamp
 	message := generateMessage(logs)
+	if rule.Reason == OOMREASON && k8sClient != nil{
+		uuid := string(uuidRegx.Find([]byte(message)))
+		uuid = strings.ReplaceAll(uuid,"_","-")
+		pl, err := k8sClient.CoreV1().Pods("").List(metav1.ListOptions{})
+		if err != nil {
+			glog.Error("Error in getting pods: %v", err.Error())
+		} else {
+			for _, pod := range pl.Items {
+				if string(pod.UID) == uuid {
+					message = fmt.Sprintf("pod was OOM killed. node:%s pod:%s namespace:%s uuid:%s",
+						pod.Spec.NodeName, pod.Name, pod.Namespace, uuid)
+					break
+				}
+			}
+		}
+	}
 	var events []types.Event
 	var changedConditions []*types.Condition
 	if rule.Type == types.Temp {