Skip to content

Commit b2ce446

Browse files
WanLinghaoWanLinghao
committed
Migrate oom watcher not relying on cAdviosr any more, it is part of plan
of removing cAdvisor from k8s. For more informations about this plan, please refer: kubernetes#68522
1 parent 6af2e9a commit b2ce446

File tree

7 files changed

+167
-34
lines changed

7 files changed

+167
-34
lines changed

‎pkg/kubelet/BUILD‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ go_library(
2121
"kubelet_pods.go",
2222
"kubelet_resources.go",
2323
"kubelet_volumes.go",
24-
"oom_watcher.go",
2524
"pod_container_deletor.go",
2625
"pod_workers.go",
2726
"reason_cache.go",
@@ -72,6 +71,7 @@ go_library(
7271
"//pkg/kubelet/network/dns:go_default_library",
7372
"//pkg/kubelet/nodelease:go_default_library",
7473
"//pkg/kubelet/nodestatus:go_default_library",
74+
"//pkg/kubelet/oom:go_default_library",
7575
"//pkg/kubelet/pleg:go_default_library",
7676
"//pkg/kubelet/pod:go_default_library",
7777
"//pkg/kubelet/preemption:go_default_library",
@@ -144,7 +144,6 @@ go_library(
144144
"//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library",
145145
"//third_party/forked/golang/expansion:go_default_library",
146146
"//vendor/github.com/golang/groupcache/lru:go_default_library",
147-
"//vendor/github.com/google/cadvisor/events:go_default_library",
148147
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
149148
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
150149
"//vendor/k8s.io/klog:go_default_library",
@@ -168,7 +167,6 @@ go_test(
168167
"kubelet_test.go",
169168
"kubelet_volumes_linux_test.go",
170169
"kubelet_volumes_test.go",
171-
"oom_watcher_test.go",
172170
"pod_container_deletor_test.go",
173171
"pod_workers_test.go",
174172
"reason_cache_test.go",
@@ -297,6 +295,7 @@ filegroup(
297295
"//pkg/kubelet/network:all-srcs",
298296
"//pkg/kubelet/nodelease:all-srcs",
299297
"//pkg/kubelet/nodestatus:all-srcs",
298+
"//pkg/kubelet/oom:all-srcs",
300299
"//pkg/kubelet/pleg:all-srcs",
301300
"//pkg/kubelet/pod:all-srcs",
302301
"//pkg/kubelet/preemption:all-srcs",

‎pkg/kubelet/kubelet.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ import (
8080
"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
8181
"k8s.io/kubernetes/pkg/kubelet/network/dns"
8282
"k8s.io/kubernetes/pkg/kubelet/nodelease"
83+
oomwatcher "k8s.io/kubernetes/pkg/kubelet/oom"
8384
"k8s.io/kubernetes/pkg/kubelet/pleg"
8485
kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
8586
"k8s.io/kubernetes/pkg/kubelet/preemption"
@@ -469,7 +470,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
469470

470471
containerRefManager := kubecontainer.NewRefManager()
471472

472-
oomWatcher := NewOOMWatcher(kubeDeps.CAdvisorInterface, kubeDeps.Recorder)
473+
oomWatcher := oomwatcher.NewOOMWatcher(kubeDeps.Recorder)
473474

474475
clusterDNS := make([]net.IP, 0, len(kubeCfg.ClusterDNS))
475476
for _, ipEntry := range kubeCfg.ClusterDNS {
@@ -1088,7 +1089,7 @@ type Kubelet struct {
10881089
os kubecontainer.OSInterface
10891090

10901091
// Watcher of out of memory events.
1091-
oomWatcher OOMWatcher
1092+
oomWatcher oomwatcher.OOMWatcher
10921093

10931094
// Monitor resource usage
10941095
resourceAnalyzer serverstats.ResourceAnalyzer

‎pkg/kubelet/oom/BUILD‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = [
6+
"oom_watcher_linux.go",
7+
"oom_watcher_unsupported.go",
8+
"types.go",
9+
],
10+
importpath = "k8s.io/kubernetes/pkg/kubelet/oom",
11+
visibility = ["//visibility:public"],
12+
deps = [
13+
"//staging/src/k8s.io/api/core/v1:go_default_library",
14+
] + select({
15+
"@io_bazel_rules_go//go/platform:android": [
16+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
17+
],
18+
"@io_bazel_rules_go//go/platform:darwin": [
19+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
20+
],
21+
"@io_bazel_rules_go//go/platform:dragonfly": [
22+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
23+
],
24+
"@io_bazel_rules_go//go/platform:freebsd": [
25+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
26+
],
27+
"@io_bazel_rules_go//go/platform:linux": [
28+
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
29+
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
30+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
31+
"//vendor/github.com/google/cadvisor/utils/oomparser:go_default_library",
32+
"//vendor/k8s.io/klog:go_default_library",
33+
],
34+
"@io_bazel_rules_go//go/platform:nacl": [
35+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
36+
],
37+
"@io_bazel_rules_go//go/platform:netbsd": [
38+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
39+
],
40+
"@io_bazel_rules_go//go/platform:openbsd": [
41+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
42+
],
43+
"@io_bazel_rules_go//go/platform:plan9": [
44+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
45+
],
46+
"@io_bazel_rules_go//go/platform:solaris": [
47+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
48+
],
49+
"@io_bazel_rules_go//go/platform:windows": [
50+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
51+
],
52+
"//conditions:default": [],
53+
}),
54+
)
55+
56+
go_test(
57+
name = "go_default_test",
58+
srcs = ["oom_watcher_linux_test.go"],
59+
embed = [":go_default_library"],
60+
deps = select({
61+
"@io_bazel_rules_go//go/platform:linux": [
62+
"//staging/src/k8s.io/api/core/v1:go_default_library",
63+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
64+
"//vendor/github.com/stretchr/testify/assert:go_default_library",
65+
],
66+
"//conditions:default": [],
67+
}),
68+
)
69+
70+
filegroup(
71+
name = "package-srcs",
72+
srcs = glob(["**"]),
73+
tags = ["automanaged"],
74+
visibility = ["//visibility:private"],
75+
)
76+
77+
filegroup(
78+
name = "all-srcs",
79+
srcs = [":package-srcs"],
80+
tags = ["automanaged"],
81+
visibility = ["//visibility:public"],
82+
)

‎pkg/kubelet/oom_watcher.go‎ renamed to ‎pkg/kubelet/oom/oom_watcher_linux.go‎

Lines changed: 18 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
// +build linux
2+
13
/*
24
Copyright 2015 The Kubernetes Authors.
35
@@ -14,61 +16,52 @@ See the License for the specific language governing permissions and
1416
limitations under the License.
1517
*/
1618

17-
package kubelet
19+
package oom
1820

1921
import (
20-
"github.com/google/cadvisor/events"
21-
cadvisorapi "github.com/google/cadvisor/info/v1"
2222
"k8s.io/api/core/v1"
2323
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2424
"k8s.io/apimachinery/pkg/util/runtime"
2525
"k8s.io/client-go/tools/record"
2626
"k8s.io/klog"
27-
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
28-
)
2927

30-
// OOMWatcher defines the interface of OOM watchers.
31-
type OOMWatcher interface {
32-
Start(ref *v1.ObjectReference) error
33-
}
28+
"github.com/google/cadvisor/utils/oomparser"
29+
)
3430

3531
type realOOMWatcher struct {
36-
cadvisor cadvisor.Interface
3732
recorder record.EventRecorder
3833
}
3934

35+
var _ OOMWatcher = &realOOMWatcher{}
36+
4037
// NewOOMWatcher creates and initializes a OOMWatcher based on parameters.
41-
func NewOOMWatcher(cadvisor cadvisor.Interface, recorder record.EventRecorder) OOMWatcher {
38+
func NewOOMWatcher(recorder record.EventRecorder) OOMWatcher {
4239
return &realOOMWatcher{
43-
cadvisor: cadvisor,
4440
recorder: recorder,
4541
}
4642
}
4743

4844
const systemOOMEvent = "SystemOOM"
4945

50-
// Watches cadvisor for system oom's and records an event for every system oom encountered.
46+
// Watches for system oom's and records an event for every system oom encountered.
5147
func (ow *realOOMWatcher) Start(ref *v1.ObjectReference) error {
52-
request := events.Request{
53-
EventType: map[cadvisorapi.EventType]bool{
54-
cadvisorapi.EventOom: true,
55-
},
56-
ContainerName: "/",
57-
IncludeSubcontainers: false,
58-
}
59-
eventChannel, err := ow.cadvisor.WatchEvents(&request)
48+
oomLog, err := oomparser.New()
6049
if err != nil {
6150
return err
6251
}
52+
outStream := make(chan *oomparser.OomInstance, 10)
53+
go oomLog.StreamOoms(outStream)
6354

6455
go func() {
6556
defer runtime.HandleCrash()
6657

67-
for event := range eventChannel.GetChannel() {
68-
klog.V(2).Infof("Got sys oom event from cadvisor: %v", event)
69-
ow.recorder.PastEventf(ref, metav1.Time{Time: event.Timestamp}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered")
58+
for event := range outStream {
59+
if event.ContainerName == "/" {
60+
klog.V(1).Infof("Got sys oom event: %v", event)
61+
ow.recorder.PastEventf(ref, metav1.Time{Time: event.TimeOfDeath}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered")
62+
}
7063
}
71-
klog.Errorf("Unexpectedly stopped receiving OOM notifications from cAdvisor")
64+
klog.Errorf("Unexpectedly stopped receiving OOM notifications")
7265
}()
7366
return nil
7467
}

‎pkg/kubelet/oom_watcher_test.go‎ renamed to ‎pkg/kubelet/oom/oom_watcher_linux_test.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package kubelet
17+
package oom
1818

1919
import (
2020
"testing"
@@ -23,14 +23,12 @@ import (
2323

2424
"k8s.io/api/core/v1"
2525
"k8s.io/client-go/tools/record"
26-
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
2726
)
2827

2928
func TestBasic(t *testing.T) {
3029
fakeRecorder := &record.FakeRecorder{}
31-
mockCadvisor := &cadvisortest.Fake{}
3230
node := &v1.ObjectReference{}
33-
oomWatcher := NewOOMWatcher(mockCadvisor, fakeRecorder)
31+
oomWatcher := NewOOMWatcher(fakeRecorder)
3432
assert.NoError(t, oomWatcher.Start(node))
3533

3634
// TODO: Improve this test once cadvisor exports events.EventChannel as an interface

‎pkg/kubelet/oom/oom_watcher_unsupported.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// +build !linux
2+
3+
/*
4+
Copyright 2019 The Kubernetes Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
package oom
19+
20+
import (
21+
"k8s.io/api/core/v1"
22+
"k8s.io/client-go/tools/record"
23+
)
24+
25+
type oomWatcherUnsupported struct{}
26+
27+
var _ OOMWatcher = new(oomWatcherUnsupported)
28+
29+
func NewOOMWatcher(_ record.EventRecorder) OOMWatcher {
30+
return &oomWatcherUnsupported{}
31+
}
32+
33+
func (ow *oomWatcherUnsupported) Start(_ *v1.ObjectReference) error {
34+
return nil
35+
}

‎pkg/kubelet/oom/types.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
package oom
17+
18+
import (
19+
"k8s.io/api/core/v1"
20+
)
21+
22+
// OOMWatcher defines the interface of OOM watchers.
23+
type OOMWatcher interface {
24+
Start(ref *v1.ObjectReference) error
25+
}

0 commit comments

Comments
 (0)