Merge pull request #10 from ruijzhan/features/oom-pod-name-new-check-fd

Features/oom pod name new check fd

Author: ringtail

Dockerfile.in

@@ -28,6 +28,7 @@ RUN test -h /etc/localtime && rm -f /etc/localtime && cp /usr/share/zoneinfo/UTC
 
 ADD ./bin/node-problem-detector /node-problem-detector
 ADD ./bin/log-counter /home/kubernetes/bin/log-counter
+ADD ./bin/check-fd /home/kubernetes/bin/check-fd
 ADD config /config
 RUN chmod +x /config/plugin/*.sh
 ENTRYPOINT ["/node-problem-detector", "--system-log-monitors=/config/kernel-monitor.json"]

Makefile

@@ -102,6 +102,14 @@ version:
 		$(BUILD_TAGS) \
 		./cmd/nodeproblemdetector
 
+./bin/check-fd: $(PKG_SOURCES)
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=linux GO111MODULE=on go build \
+		-mod vendor \
+		-o bin/check-fd \
+		-ldflags '-X $(PKG)/pkg/version.version=$(VERSION)' \
+		$(BUILD_TAGS) \
+		./cmd/checkfd
+
 Dockerfile: Dockerfile.in
 	sed -e 's|@BASEIMAGE@|$(BASEIMAGE)|g' $< >$@
 
@@ -118,12 +126,12 @@ e2e-test: vet fmt build-tar
 	-boskos-project-type=$(BOSKOS_PROJECT_TYPE) -job-name=$(JOB_NAME) \
 	-artifacts-dir=$(ARTIFACTS)
 
-build-binaries: ./bin/node-problem-detector ./bin/log-counter
+build-binaries: ./bin/node-problem-detector ./bin/log-counter ./bin/check-fd
 
 build-container: build-binaries Dockerfile
 	docker build -t $(IMAGE) .
 
-build-tar: ./bin/node-problem-detector ./bin/log-counter
+build-tar: ./bin/node-problem-detector ./bin/log-counter ./bin/check-fd
 	tar -zcvf $(TARBALL) bin/ config/ test/e2e-install.sh
 	sha1sum $(TARBALL)
 	md5sum $(TARBALL)
@@ -150,4 +158,5 @@ push: push-container push-tar
 clean:
 	rm -f bin/log-counter
 	rm -f bin/node-problem-detector
+	rm -f bin/check-fd
 	rm -f node-problem-detector-*.tar.gz

cmd/checkfd/check_fd.go

@@ -0,0 +1,111 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+const (
+	OK      = 0
+	NONOK   = 1
+	UNKNOWN = 2
+)
+
+var (
+	procPath           string
+	printCount         bool
+	percentThreshold   int
+)
+
+func init() {
+	flag.StringVar(&procPath, "p", "/proc", "actual path of /proc")
+	flag.BoolVar(&printCount,"c", false, "print numbers of used df and max")
+	flag.IntVar(&percentThreshold, "t", 80, "Warning threshold of percentage of fd usage to max limitation")
+}
+
+func main() {
+	flag.Parse()
+
+	if percentThreshold >= 100 || percentThreshold <=0 {
+		fmt.Fprintf(os.Stderr, "value of -t must between 0 and 100")
+		os.Exit(UNKNOWN)
+	}
+
+	maxPath := procPath + "/sys/fs/file-max"
+	f, err := os.Open(maxPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v", err)
+		os.Exit(UNKNOWN)
+	}
+	defer f.Close()
+	maxBytes, err := ioutil.ReadAll(f)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v", err)
+		os.Exit(UNKNOWN)
+	}
+	fdMax, err := strconv.Atoi(strings.TrimSpace(string(maxBytes)))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v", err)
+		os.Exit(UNKNOWN)
+	}
+
+	files, err := ioutil.ReadDir(procPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v", err)
+		os.Exit(UNKNOWN)
+	}
+
+	ch := make(chan int)
+	wg := sync.WaitGroup{}
+	re := regexp.MustCompile(`[0-9][0-9]*`)
+	for _, f := range files {
+		if f.IsDir() {
+			if re.MatchString(f.Name()) {
+				wg.Add(1)
+				go countFD(procPath+"/"+f.Name()+"/fd", &wg, ch)
+			}
+		}
+	}
+
+	go func() {
+		wg.Wait()
+		close(ch)
+	}()
+
+	fdTotal := 0
+	for {
+		c, ok := <-ch
+		if !ok {
+			break
+		}
+		fdTotal += c
+	}
+
+	if printCount {
+		fmt.Fprintf(os.Stdout, "current fd usage is %d and limitaion is %d\n", fdTotal, fdMax)
+	}
+	if fdTotal > fdMax / 100 * percentThreshold {
+		fmt.Fprintf(os.Stdout, "current fd usage is %d and is over %d of limition %d, \n",
+			fdTotal, percentThreshold, fdMax)
+		os.Exit(NONOK)
+	} else {
+		fmt.Fprintf(os.Stdout, "node has no fd pressure\n")
+		os.Exit(OK)
+	}
+}
+
+func countFD(path string, wg *sync.WaitGroup, fNum chan<- int) {
+	defer wg.Done()
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v", err)
+		return
+	}
+	fNum <- len(files)
+}

cmd/nodeproblemdetector/node_problem_detector.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"k8s.io/node-problem-detector/pkg/systemlogmonitor"
 	"os"
 
 	"github.com/golang/glog"
@@ -55,6 +56,11 @@ func main() {
 		glog.Fatalf("No problem daemon is configured")
 	}
 
+	if c := systemlogmonitor.InitK8sClientOrDie(npdo); c != nil {
+		glog.Info("System Log Monitor K8S client initialized")
+	} else {
+		glog.Error("Failed to initialize System Log Monitor K8S client")
+	}
 	// Initialize exporters.
 	defaultExporters := []types.Exporter{}
 	if ke := k8sexporter.NewExporterOrDie(npdo); ke != nil {

config/plugin/network_problem.sh

@@ -1,23 +1,36 @@
 #!/bin/bash
 
-# This plugin checks for common network issues. Currently, it only checks
-# if the conntrack table is full. 
+# This plugin checks for common network issues.
+# Currently only checks if conntrack table is more than 90% used.
 
-OK=0
-NONOK=1
-UNKNOWN=2
+readonly OK=0
+readonly NONOK=1
+readonly UNKNOWN=2
 
-[ -f /proc/sys/net/ipv4/netfilter/ip_conntrack_max ] || exit $UNKNOWN
-[ -f /proc/sys/net/ipv4/netfilter/ip_conntrack_count ] || exit $UNKNOWN
+# "nf_conntrack" replaces "ip_conntrack" - support both
+readonly NF_CT_COUNT_PATH='/proc/sys/net/netfilter/nf_conntrack_count'
+readonly NF_CT_MAX_PATH='/proc/sys/net/netfilter/nf_conntrack_max'
+readonly IP_CT_COUNT_PATH='/proc/sys/net/ipv4/netfilter/ip_conntrack_count'
+readonly IP_CT_MAX_PATH='/proc/sys/net/ipv4/netfilter/ip_conntrack_max'
 
-conntrack_max=$(cat /proc/sys/net/ipv4/netfilter/ip_conntrack_max)
-conntrack_count=$(cat /proc/sys/net/ipv4/netfilter/ip_conntrack_count)
-
-if (( conntrack_count >= conntrack_max )); then
-    echo "Conntrack table full"
-    exit $NONOK
+if [[ -f $NF_CT_COUNT_PATH ]] && [[ -f $NF_CT_MAX_PATH ]]; then
+  readonly CT_COUNT_PATH=$NF_CT_COUNT_PATH
+  readonly CT_MAX_PATH=$NF_CT_MAX_PATH
+elif [[ -f $IP_CT_COUNT_PATH ]] && [[ -f $IP_CT_MAX_PATH ]]; then
+  readonly CT_COUNT_PATH=$IP_CT_COUNT_PATH
+  readonly CT_MAX_PATH=$IP_CT_MAX_PATH
+else
+  exit $UNKNOWN
 fi
 
-echo "Conntrack table available"
-exit $OK
+readonly conntrack_count=$(< $CT_COUNT_PATH) || exit $UNKNOWN
+readonly conntrack_max=$(< $CT_MAX_PATH) || exit $UNKNOWN
+readonly conntrack_usage_msg="${conntrack_count} out of ${conntrack_max}"
 
+if (( conntrack_count > conntrack_max * 9 /10 )); then
+  echo "Conntrack table usage over 90%: ${conntrack_usage_msg}"
+  exit $NONOK
+else
+  echo "Conntrack table usage: ${conntrack_usage_msg}"
+  exit $OK
+fi

pkg/systemlogmonitor/log_monitor.go

@@ -19,7 +19,16 @@ package systemlogmonitor
 import (
 	"encoding/json"
 	"io/ioutil"
+	"k8s.io/heapster/common/kubernetes"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/node-problem-detector/cmd/options"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
 	"time"
+	"fmt"
 
 	"github.com/golang/glog"
 
@@ -32,9 +41,19 @@ import (
 	"k8s.io/node-problem-detector/pkg/types"
 	"k8s.io/node-problem-detector/pkg/util"
 	"k8s.io/node-problem-detector/pkg/util/tomb"
+	"k8s.io/node-problem-detector/pkg/version"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const SystemLogMonitorName = "system-log-monitor"
+const (
+	SystemLogMonitorName = "system-log-monitor"
+	OOMREASON            = "PodOOMKilling"
+)
+
+var (
+	uuidRegx                      *regexp.Regexp
+	k8sClient                     *clientset.Clientset
+)
 
 func init() {
 	problemdaemon.Register(
@@ -44,6 +63,10 @@ func init() {
 			CmdOptionDescription:     "Set to config file paths."})
 }
 
+func init() {
+	uuidRegx = regexp.MustCompile("[0-9a-f]{8}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{12}")
+}
+
 type logMonitor struct {
 	configPath string
 	watcher    watchertypes.LogWatcher
@@ -55,6 +78,17 @@ type logMonitor struct {
 	tomb       *tomb.Tomb
 }
 
+func InitK8sClientOrDie(options *options.NodeProblemDetectorOptions) *clientset.Clientset{
+	uri, _ := url.Parse(options.ApiServerOverride)
+	cfg, err := kubernetes.GetKubeClientConfig(uri)
+	if err != nil {
+		panic(err)
+	}
+	cfg.UserAgent = fmt.Sprintf("%s/%s", filepath.Base(os.Args[0]), version.Version())
+	k8sClient = clientset.NewForConfigOrDie(cfg)
+	return k8sClient
+}
+
 // NewLogMonitorOrDie create a new LogMonitor, panic if error occurs.
 func NewLogMonitorOrDie(configPath string) types.Monitor {
 	l := &logMonitor{
@@ -167,6 +201,22 @@ func (l *logMonitor) generateStatus(logs []*logtypes.Log, rule systemlogtypes.Ru
 	// We use the timestamp of the first log line as the timestamp of the status.
 	timestamp := logs[0].Timestamp
 	message := generateMessage(logs)
+	if rule.Reason == OOMREASON && k8sClient != nil{
+		uuid := string(uuidRegx.Find([]byte(message)))
+		uuid = strings.ReplaceAll(uuid,"_","-")
+		pl, err := k8sClient.CoreV1().Pods("").List(metav1.ListOptions{})
+		if err != nil {
+			glog.Error("Error in getting pods: %v", err.Error())
+		} else {
+			for _, pod := range pl.Items {
+				if string(pod.UID) == uuid {
+					message = fmt.Sprintf("pod was OOM killed. node:%s pod:%s namespace:%s uuid:%s",
+						pod.Spec.NodeName, pod.Name, pod.Namespace, uuid)
+					break
+				}
+			}
+		}
+	}
 	var events []types.Event
 	var changedConditions []*types.Condition
 	if rule.Type == types.Temp {