Skip to content

Commit 8ec6167

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#74942 from WanLinghao/event_replace
Migrate oom watcher not relying on cAdviosr's API any more
2 parents 3ba70b9 + 611563c commit 8ec6167

File tree

8 files changed

+171
-34
lines changed

8 files changed

+171
-34
lines changed

‎hack/.golint_failures‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,7 @@ pkg/kubelet/dockershim/network/testing
186186
pkg/kubelet/events
187187
pkg/kubelet/lifecycle
188188
pkg/kubelet/metrics
189+
pkg/kubelet/oom
189190
pkg/kubelet/pod
190191
pkg/kubelet/pod/testing
191192
pkg/kubelet/preemption

‎pkg/kubelet/BUILD‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ go_library(
2121
"kubelet_pods.go",
2222
"kubelet_resources.go",
2323
"kubelet_volumes.go",
24-
"oom_watcher.go",
2524
"pod_container_deletor.go",
2625
"pod_workers.go",
2726
"reason_cache.go",
@@ -70,6 +69,7 @@ go_library(
7069
"//pkg/kubelet/network/dns:go_default_library",
7170
"//pkg/kubelet/nodelease:go_default_library",
7271
"//pkg/kubelet/nodestatus:go_default_library",
72+
"//pkg/kubelet/oom:go_default_library",
7373
"//pkg/kubelet/pleg:go_default_library",
7474
"//pkg/kubelet/pod:go_default_library",
7575
"//pkg/kubelet/preemption:go_default_library",
@@ -145,7 +145,6 @@ go_library(
145145
"//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library",
146146
"//third_party/forked/golang/expansion:go_default_library",
147147
"//vendor/github.com/golang/groupcache/lru:go_default_library",
148-
"//vendor/github.com/google/cadvisor/events:go_default_library",
149148
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
150149
"//vendor/k8s.io/klog:go_default_library",
151150
"//vendor/k8s.io/utils/exec:go_default_library",
@@ -168,7 +167,6 @@ go_test(
168167
"kubelet_test.go",
169168
"kubelet_volumes_linux_test.go",
170169
"kubelet_volumes_test.go",
171-
"oom_watcher_test.go",
172170
"pod_container_deletor_test.go",
173171
"pod_workers_test.go",
174172
"reason_cache_test.go",
@@ -297,6 +295,7 @@ filegroup(
297295
"//pkg/kubelet/network:all-srcs",
298296
"//pkg/kubelet/nodelease:all-srcs",
299297
"//pkg/kubelet/nodestatus:all-srcs",
298+
"//pkg/kubelet/oom:all-srcs",
300299
"//pkg/kubelet/pleg:all-srcs",
301300
"//pkg/kubelet/pod:all-srcs",
302301
"//pkg/kubelet/preemption:all-srcs",

‎pkg/kubelet/kubelet.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ import (
7878
"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
7979
"k8s.io/kubernetes/pkg/kubelet/network/dns"
8080
"k8s.io/kubernetes/pkg/kubelet/nodelease"
81+
oomwatcher "k8s.io/kubernetes/pkg/kubelet/oom"
8182
"k8s.io/kubernetes/pkg/kubelet/pleg"
8283
kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
8384
"k8s.io/kubernetes/pkg/kubelet/preemption"
@@ -464,7 +465,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
464465

465466
containerRefManager := kubecontainer.NewRefManager()
466467

467-
oomWatcher := NewOOMWatcher(kubeDeps.CAdvisorInterface, kubeDeps.Recorder)
468+
oomWatcher := oomwatcher.NewOOMWatcher(kubeDeps.Recorder)
468469

469470
clusterDNS := make([]net.IP, 0, len(kubeCfg.ClusterDNS))
470471
for _, ipEntry := range kubeCfg.ClusterDNS {
@@ -1078,7 +1079,7 @@ type Kubelet struct {
10781079
os kubecontainer.OSInterface
10791080

10801081
// Watcher of out of memory events.
1081-
oomWatcher OOMWatcher
1082+
oomWatcher oomwatcher.OOMWatcher
10821083

10831084
// Monitor resource usage
10841085
resourceAnalyzer serverstats.ResourceAnalyzer

‎pkg/kubelet/oom/BUILD‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = [
6+
"oom_watcher_linux.go",
7+
"oom_watcher_unsupported.go",
8+
"types.go",
9+
],
10+
importpath = "k8s.io/kubernetes/pkg/kubelet/oom",
11+
visibility = ["//visibility:public"],
12+
deps = [
13+
"//staging/src/k8s.io/api/core/v1:go_default_library",
14+
] + select({
15+
"@io_bazel_rules_go//go/platform:android": [
16+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
17+
],
18+
"@io_bazel_rules_go//go/platform:darwin": [
19+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
20+
],
21+
"@io_bazel_rules_go//go/platform:dragonfly": [
22+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
23+
],
24+
"@io_bazel_rules_go//go/platform:freebsd": [
25+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
26+
],
27+
"@io_bazel_rules_go//go/platform:linux": [
28+
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
29+
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
30+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
31+
"//vendor/github.com/google/cadvisor/utils/oomparser:go_default_library",
32+
"//vendor/k8s.io/klog:go_default_library",
33+
],
34+
"@io_bazel_rules_go//go/platform:nacl": [
35+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
36+
],
37+
"@io_bazel_rules_go//go/platform:netbsd": [
38+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
39+
],
40+
"@io_bazel_rules_go//go/platform:openbsd": [
41+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
42+
],
43+
"@io_bazel_rules_go//go/platform:plan9": [
44+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
45+
],
46+
"@io_bazel_rules_go//go/platform:solaris": [
47+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
48+
],
49+
"@io_bazel_rules_go//go/platform:windows": [
50+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
51+
],
52+
"//conditions:default": [],
53+
}),
54+
)
55+
56+
go_test(
57+
name = "go_default_test",
58+
srcs = ["oom_watcher_linux_test.go"],
59+
embed = [":go_default_library"],
60+
deps = select({
61+
"@io_bazel_rules_go//go/platform:linux": [
62+
"//staging/src/k8s.io/api/core/v1:go_default_library",
63+
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
64+
"//vendor/github.com/stretchr/testify/assert:go_default_library",
65+
],
66+
"//conditions:default": [],
67+
}),
68+
)
69+
70+
filegroup(
71+
name = "package-srcs",
72+
srcs = glob(["**"]),
73+
tags = ["automanaged"],
74+
visibility = ["//visibility:private"],
75+
)
76+
77+
filegroup(
78+
name = "all-srcs",
79+
srcs = [":package-srcs"],
80+
tags = ["automanaged"],
81+
visibility = ["//visibility:public"],
82+
)

‎pkg/kubelet/oom_watcher.go‎ renamed to ‎pkg/kubelet/oom/oom_watcher_linux.go‎

Lines changed: 18 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
// +build linux
2+
13
/*
24
Copyright 2015 The Kubernetes Authors.
35
@@ -14,61 +16,52 @@ See the License for the specific language governing permissions and
1416
limitations under the License.
1517
*/
1618

17-
package kubelet
19+
package oom
1820

1921
import (
20-
"github.com/google/cadvisor/events"
21-
cadvisorapi "github.com/google/cadvisor/info/v1"
2222
"k8s.io/api/core/v1"
2323
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2424
"k8s.io/apimachinery/pkg/util/runtime"
2525
"k8s.io/client-go/tools/record"
2626
"k8s.io/klog"
27-
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
28-
)
2927

30-
// OOMWatcher defines the interface of OOM watchers.
31-
type OOMWatcher interface {
32-
Start(ref *v1.ObjectReference) error
33-
}
28+
"github.com/google/cadvisor/utils/oomparser"
29+
)
3430

3531
type realOOMWatcher struct {
36-
cadvisor cadvisor.Interface
3732
recorder record.EventRecorder
3833
}
3934

35+
var _ OOMWatcher = &realOOMWatcher{}
36+
4037
// NewOOMWatcher creates and initializes a OOMWatcher based on parameters.
41-
func NewOOMWatcher(cadvisor cadvisor.Interface, recorder record.EventRecorder) OOMWatcher {
38+
func NewOOMWatcher(recorder record.EventRecorder) OOMWatcher {
4239
return &realOOMWatcher{
43-
cadvisor: cadvisor,
4440
recorder: recorder,
4541
}
4642
}
4743

4844
const systemOOMEvent = "SystemOOM"
4945

50-
// Watches cadvisor for system oom's and records an event for every system oom encountered.
46+
// Watches for system oom's and records an event for every system oom encountered.
5147
func (ow *realOOMWatcher) Start(ref *v1.ObjectReference) error {
52-
request := events.Request{
53-
EventType: map[cadvisorapi.EventType]bool{
54-
cadvisorapi.EventOom: true,
55-
},
56-
ContainerName: "/",
57-
IncludeSubcontainers: false,
58-
}
59-
eventChannel, err := ow.cadvisor.WatchEvents(&request)
48+
oomLog, err := oomparser.New()
6049
if err != nil {
6150
return err
6251
}
52+
outStream := make(chan *oomparser.OomInstance, 10)
53+
go oomLog.StreamOoms(outStream)
6354

6455
go func() {
6556
defer runtime.HandleCrash()
6657

67-
for event := range eventChannel.GetChannel() {
68-
klog.V(2).Infof("Got sys oom event from cadvisor: %v", event)
69-
ow.recorder.PastEventf(ref, metav1.Time{Time: event.Timestamp}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered")
58+
for event := range outStream {
59+
if event.ContainerName == "/" {
60+
klog.V(1).Infof("Got sys oom event: %v", event)
61+
ow.recorder.PastEventf(ref, metav1.Time{Time: event.TimeOfDeath}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered")
62+
}
7063
}
71-
klog.Errorf("Unexpectedly stopped receiving OOM notifications from cAdvisor")
64+
klog.Errorf("Unexpectedly stopped receiving OOM notifications")
7265
}()
7366
return nil
7467
}

‎pkg/kubelet/oom_watcher_test.go‎ renamed to ‎pkg/kubelet/oom/oom_watcher_linux_test.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package kubelet
17+
package oom
1818

1919
import (
2020
"testing"
@@ -23,14 +23,12 @@ import (
2323

2424
"k8s.io/api/core/v1"
2525
"k8s.io/client-go/tools/record"
26-
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
2726
)
2827

2928
func TestBasic(t *testing.T) {
3029
fakeRecorder := &record.FakeRecorder{}
31-
mockCadvisor := &cadvisortest.Fake{}
3230
node := &v1.ObjectReference{}
33-
oomWatcher := NewOOMWatcher(mockCadvisor, fakeRecorder)
31+
oomWatcher := NewOOMWatcher(fakeRecorder)
3432
assert.NoError(t, oomWatcher.Start(node))
3533

3634
// TODO: Improve this test once cadvisor exports events.EventChannel as an interface

‎pkg/kubelet/oom/oom_watcher_unsupported.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// +build !linux
2+
3+
/*
4+
Copyright 2019 The Kubernetes Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package oom
20+
21+
import (
22+
"k8s.io/api/core/v1"
23+
"k8s.io/client-go/tools/record"
24+
)
25+
26+
type oomWatcherUnsupported struct{}
27+
28+
var _ OOMWatcher = new(oomWatcherUnsupported)
29+
30+
// NewOOMWatcher creates a fake one here
31+
func NewOOMWatcher(_ record.EventRecorder) OOMWatcher {
32+
return &oomWatcherUnsupported{}
33+
}
34+
35+
func (ow *oomWatcherUnsupported) Start(_ *v1.ObjectReference) error {
36+
return nil
37+
}

‎pkg/kubelet/oom/types.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package oom
18+
19+
import (
20+
"k8s.io/api/core/v1"
21+
)
22+
23+
// OOMWatcher defines the interface of OOM watchers.
24+
type OOMWatcher interface {
25+
Start(ref *v1.ObjectReference) error
26+
}

0 commit comments

Comments
 (0)