Merge pull request #1 from sapcc/container-id

use container ID instead of pod UID when getting info from docker

Author: jknipper

.dockerignore

@@ -0,0 +1 @@
+vendor/

.travis.yml

@@ -1,7 +1,6 @@
 sudo: required
-language: go
 services:
   - docker
 install: true
 
-script: make container test vet
+script: make container

Dockerfile

@@ -1,8 +1,22 @@
+FROM golang:1.11
+
+WORKDIR /go/src/github.com/sapcc/kuberntes-oomkill-exporter
+ENV GO111MODULE=on \
+    CGOENABLED=0
+ADD go.mod go.sum ./
+RUN go mod download
+ADD cache/main.go .
+RUN go build -v -o /dev/null
+ADD . .
+RUN go build -v -o /kubernetes-oomkill-exporter
+RUN go test -v
+RUN go vet
+
 FROM alpine:3.8
 LABEL maintainer="jan.knipper@sap.com"
 
 RUN apk --no-cache add ca-certificates
-COPY kubernetes-oomkill-exporter /kubernetes-oomkill-exporter
+COPY --from=0 /kubernetes-oomkill-exporter /kubernetes-oomkill-exporter
 
 ENTRYPOINT ["/kubernetes-oomkill-exporter"]
 CMD ["-logtostderr"]

Makefile

@@ -1,40 +1,19 @@
 TARGET = kubernetes-oomkill-exporter
 GOTARGET = github.com/sapcc/$(TARGET)
-BUILDMNT = /go/src/$(GOTARGET)
 REGISTRY ?= sapcc
-VERSION ?= 0.2.0
+VERSION ?= 0.3.0
 IMAGE = $(REGISTRY)/$(BIN)
-BUILD_IMAGE ?= golang:1.11-alpine3.8
 DOCKER ?= docker
-DIR := ${CURDIR}
-
-ifneq ($(VERBOSE),)
-VERBOSE_FLAG = -v
-endif
-TESTARGS ?= $(VERBOSE_FLAG) -timeout 60s
-TEST_PKGS ?= $(GOTARGET)/...
-TEST = CGO_ENABLED=0 go test $(TEST_PKGS) $(TESTARGS)
-VET_PKGS ?= $(GOTARGET)/...
-VET = CGO_ENABLED=0 go vet $(VET_PKGS)
-
-DOCKER_BUILD ?= $(DOCKER) run --rm -v $(DIR):$(BUILDMNT) -w $(BUILDMNT) $(BUILD_IMAGE) /bin/sh -c
 
 all: container
 
 container:
-	$(DOCKER_BUILD) 'go build'
 	$(DOCKER) build -t $(REGISTRY)/$(TARGET):latest -t $(REGISTRY)/$(TARGET):$(VERSION) .
 
 push:
 	$(DOCKER) push $(REGISTRY)/$(TARGET):latest
 	$(DOCKER) push $(REGISTRY)/$(TARGET):$(VERSION)
 
-test:
-	$(DOCKER_BUILD) '$(TEST)'
-
-vet:
-	$(DOCKER_BUILD) '$(VET)'
-
 .PHONY: all local container push
 
 clean:

cache/main.go

@@ -0,0 +1,25 @@
+//This file is only used to speedup the docker build
+//We use this to download and compile the go module dependencies before adding our own source code.
+//See Dockerfile for more details
+
+package main
+
+import (
+	_ "flag"
+	_ "net/http"
+	_ "regexp"
+	_ "strings"
+
+	_ "docker.io/go-docker"
+	_ "docker.io/go-docker/api/types"
+	_ "github.com/golang/glog"
+	_ "github.com/prometheus/client_golang/prometheus"
+	_ "github.com/prometheus/client_golang/prometheus/promhttp"
+	_ "golang.org/x/net/context"
+	_ "k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/kmsg"
+	_ "k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/types"
+)
+
+func main() {
+
+}

main.go

@@ -2,14 +2,12 @@ package main
 
 import (
 	"flag"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
 
 	docker_client "docker.io/go-docker"
 	docker_types "docker.io/go-docker/api/types"
-	docker_filters "docker.io/go-docker/api/types/filters"
 	"github.com/golang/glog"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -18,10 +16,8 @@ import (
 	"k8s.io/node-problem-detector/pkg/systemlogmonitor/logwatchers/types"
 )
 
-const (
-	OOMMatchExpression   = ".*killed as a result of limit of.*"
-	PodExtractExpression = "^.+/pod(\\w+\\-\\w+\\-\\w+\\-\\w+\\-\\w+)/.+$"
-	PodUIDLabel          = "io.kubernetes.pod.uid"
+var (
+	kmesgRE = regexp.MustCompile("/pod(\\w+\\-\\w+\\-\\w+\\-\\w+\\-\\w+)/([a-f0-9]+) killed as a result of limit of /kubepods")
 )
 
 var (
@@ -74,66 +70,43 @@ func main() {
 	}
 
 	for log := range logCh {
-		podUID := getPodUIDFromLog(log.Message)
-		if podUID != "" {
-			container, err := getContainerFromPod(podUID, dockerClient)
-
+		podUID, containerID := getContainerIDFromLog(log.Message)
+		if containerID != "" {
+			container, err := getContainer(containerID, dockerClient)
 			if err != nil {
-				glog.Warningf("Could not get container for pod UID %s: %v", podUID, err)
+				glog.Warningf("Could not get container %s for pod %s: %v", containerID, podUID, err)
 			} else {
-				prometheusCount(container)
+				prometheusCount(container.Config.Labels)
 			}
 		}
 	}
 }
 
-func getPodUIDFromLog(log string) string {
-	match, err := regexp.MatchString(OOMMatchExpression, log)
-	if err != nil {
-		return ""
-	}
-
-	var ret []string
-	if match {
-		re := regexp.MustCompile(PodExtractExpression)
-		ret = re.FindStringSubmatch(log)
-		if len(ret) == 2 {
-			return ret[1]
-		}
+func getContainerIDFromLog(log string) (string, string) {
+	if matches := kmesgRE.FindStringSubmatch(log); matches != nil {
+		return matches[1], matches[2]
 	}
 
-	return ""
+	return "", ""
 }
 
-func getContainerFromPod(podUID string, cli *docker_client.Client) (docker_types.Container, error) {
-	filters := docker_filters.NewArgs()
-	filters.Add("label", fmt.Sprintf("%s=%s", PodUIDLabel, podUID))
-	filters.Add("label", fmt.Sprintf("%s=%s", "io.kubernetes.docker.type", "container"))
-
-	listOpts := docker_types.ContainerListOptions{
-		Filters: filters,
-	}
-
-	containers, err := cli.ContainerList(context.Background(), listOpts)
+func getContainer(containerID string, cli *docker_client.Client) (docker_types.ContainerJSON, error) {
+	container, err := cli.ContainerInspect(context.Background(), containerID)
 	if err != nil {
-		return docker_types.Container{}, err
-	}
-
-	if len(containers) < 1 {
-		return docker_types.Container{}, fmt.Errorf("There should be at least one container with UID %s", podUID)
+		return docker_types.ContainerJSON{}, err
 	}
+	return container, nil
 
-	return containers[0], nil
 }
 
-func prometheusCount(container docker_types.Container) {
+func prometheusCount(containerLabels map[string]string) {
 	var counter prometheus.Counter
 	var err error
 
 	var labels map[string]string
 	labels = make(map[string]string)
 	for key, label := range prometheusContainerLabels {
-		labels[label] = container.Labels[key]
+		labels[label] = containerLabels[key]
 	}
 
 	glog.V(5).Infof("Labels: %v\n", labels)

main_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"reflect"
 	"strings"
 	"testing"
 
@@ -11,17 +10,22 @@ import (
 )
 
 func TestGetPodUIDFromLog(t *testing.T) {
-	klog, podUIDs := getTestData()
-	var extractedUIDs []string
+	klog, podUIDs, containerIDs := getTestData()
+
+	var extractedContainerIDs []string
+	var extractedPodUIDs []string
 
 	for _, msg := range klog {
 		parsedMsg, err := parseMessage(msg)
 		require.NoError(t, err, "There should be no error while parsing kernel log")
-		uid := getPodUIDFromLog(parsedMsg.Message)
-		extractedUIDs = append(extractedUIDs, uid)
+		uid, cid := getContainerIDFromLog(parsedMsg.Message)
+		fmt.Println(uid)
+		extractedContainerIDs = append(extractedContainerIDs, cid)
+		extractedPodUIDs = append(extractedPodUIDs, uid)
 	}
 
-	require.True(t, reflect.DeepEqual(podUIDs, extractedUIDs), "Extracted UIDs do not match the expected result")
+	require.Equal(t, containerIDs, extractedContainerIDs, "Extracted container ids do not match the expected result")
+	require.Equal(t, podUIDs, extractedPodUIDs, "Extracted container ids do not match the expected result")
 }
 
 func parseMessage(input string) (kmsgparser.Message, error) {
@@ -44,13 +48,17 @@ func parseMessage(input string) (kmsgparser.Message, error) {
 	}, nil
 }
 
-func getTestData() ([]string, []string) {
+func getTestData() ([]string, []string, []string) {
 	return []string{
 			"6,22743,6115623303887,-;Task in /kubepods/burstable/pode501ca8a-ec23-11e8-b17a-0a586444015a/f24766bce80e0ce4f0ca2887da2be9d0d250448d7ef503d9f85bf5e549c757d5 killed as a result of limit of /kubepods/burstable/pode501ca8a-ec23-11e8-b17a-0a586444015a",
 			"6,23800,6780904484233,-;Task in /kubepods/burstable/pod0c4e2576-ef09-11e8-b17a-0a586444015a/9df959ad4292532c5d551226063bd840b906cbf118983fffefa0e3ab90331dc2 killed as a result of limit of /kubepods/burstable/pod0c4e2576-ef09-11e8-b17a-0a586444015a/9df959ad4292532c5d551226063bd840b906cbf118983fffefa0e3ab90331dc2",
 		},
 		[]string{
 			"e501ca8a-ec23-11e8-b17a-0a586444015a",
 			"0c4e2576-ef09-11e8-b17a-0a586444015a",
+		},
+		[]string{
+			"f24766bce80e0ce4f0ca2887da2be9d0d250448d7ef503d9f85bf5e549c757d5",
+			"9df959ad4292532c5d551226063bd840b906cbf118983fffefa0e3ab90331dc2",
 		}
 }