Skip to content

Commit 1008f22

Browse files
qiutongsbobbypage
qiutongs
authored and
bobbypage
committed
Add missing file system metrics in containerd handler
Signed-off-by: Qiutong Song <[email protected]>
1 parent 8dcedd2 commit 1008f22

File tree

8 files changed

+160
-54
lines changed

8 files changed

+160
-54
lines changed

‎container/common/fsHandler.go

Lines changed: 82 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,21 @@ type FsUsage struct {
3737
InodeUsage uint64
3838
}
3939

40+
type FsUsageProvider interface {
41+
// Usage returns the fs usage
42+
Usage() (*FsUsage, error)
43+
// Targets returns where the fs usage metric is collected,it maybe a directory ,a file or some
44+
// information about the snapshotter(for containerd)
45+
Targets() []string
46+
}
47+
4048
type realFsHandler struct {
4149
sync.RWMutex
42-
lastUpdate time.Time
43-
usage FsUsage
44-
period time.Duration
45-
minPeriod time.Duration
46-
rootfs string
47-
extraDir string
48-
fsInfo fs.FsInfo
50+
lastUpdate time.Time
51+
usage FsUsage
52+
period time.Duration
53+
minPeriod time.Duration
54+
usageProvider FsUsageProvider
4955
// Tells the container to stop.
5056
stopChan chan struct{}
5157
}
@@ -58,56 +64,33 @@ const DefaultPeriod = time.Minute
5864

5965
var _ FsHandler = &realFsHandler{}
6066

61-
func NewFsHandler(period time.Duration, rootfs, extraDir string, fsInfo fs.FsInfo) FsHandler {
67+
func NewFsHandler(period time.Duration, provider FsUsageProvider) FsHandler {
6268
return &realFsHandler{
63-
lastUpdate: time.Time{},
64-
usage: FsUsage{},
65-
period: period,
66-
minPeriod: period,
67-
rootfs: rootfs,
68-
extraDir: extraDir,
69-
fsInfo: fsInfo,
70-
stopChan: make(chan struct{}, 1),
69+
lastUpdate: time.Time{},
70+
usage: FsUsage{},
71+
period: period,
72+
minPeriod: period,
73+
usageProvider: provider,
74+
stopChan: make(chan struct{}, 1),
7175
}
7276
}
7377

7478
func (fh *realFsHandler) update() error {
75-
var (
76-
rootUsage, extraUsage fs.UsageInfo
77-
rootErr, extraErr error
78-
)
79-
// TODO(vishh): Add support for external mounts.
80-
if fh.rootfs != "" {
81-
rootUsage, rootErr = fh.fsInfo.GetDirUsage(fh.rootfs)
82-
}
8379

84-
if fh.extraDir != "" {
85-
extraUsage, extraErr = fh.fsInfo.GetDirUsage(fh.extraDir)
80+
usage, err := fh.usageProvider.Usage()
81+
82+
if err != nil {
83+
return err
8684
}
8785

88-
// Wait to handle errors until after all operartions are run.
89-
// An error in one will not cause an early return, skipping others
9086
fh.Lock()
9187
defer fh.Unlock()
9288
fh.lastUpdate = time.Now()
93-
if fh.rootfs != "" && rootErr == nil {
94-
fh.usage.InodeUsage = rootUsage.Inodes
95-
fh.usage.BaseUsageBytes = rootUsage.Bytes
96-
fh.usage.TotalUsageBytes = rootUsage.Bytes
97-
}
98-
if fh.extraDir != "" && extraErr == nil {
99-
if fh.rootfs != "" {
100-
fh.usage.TotalUsageBytes += extraUsage.Bytes
101-
} else {
102-
// rootfs is empty, totalUsageBytes use extra usage bytes
103-
fh.usage.TotalUsageBytes = extraUsage.Bytes
104-
}
105-
}
10689

107-
// Combine errors into a single error to return
108-
if rootErr != nil || extraErr != nil {
109-
return fmt.Errorf("rootDiskErr: %v, extraDiskErr: %v", rootErr, extraErr)
110-
}
90+
fh.usage.InodeUsage = usage.InodeUsage
91+
fh.usage.BaseUsageBytes = usage.BaseUsageBytes
92+
fh.usage.TotalUsageBytes = usage.TotalUsageBytes
93+
11194
return nil
11295
}
11396

@@ -130,7 +113,8 @@ func (fh *realFsHandler) trackUsage() {
130113
// if the long duration is persistent either because of slow
131114
// disk or lots of containers.
132115
longOp = longOp + time.Second
133-
klog.V(2).Infof("fs: disk usage and inodes count on following dirs took %v: %v; will not log again for this container unless duration exceeds %v", duration, []string{fh.rootfs, fh.extraDir}, longOp)
116+
klog.V(2).Infof(`fs: disk usage and inodes count on targets took %v: %v; `+
117+
`will not log again for this container unless duration exceeds %v`, duration, fh.usageProvider.Targets(), longOp)
134118
}
135119
select {
136120
case <-fh.stopChan:
@@ -153,3 +137,55 @@ func (fh *realFsHandler) Usage() FsUsage {
153137
defer fh.RUnlock()
154138
return fh.usage
155139
}
140+
141+
type fsUsageProvider struct {
142+
fsInfo fs.FsInfo
143+
rootFs string
144+
extraDir string
145+
}
146+
147+
func NewGeneralFsUsageProvider(fsInfo fs.FsInfo, rootFs, extraDir string) FsUsageProvider {
148+
return &fsUsageProvider{
149+
fsInfo: fsInfo,
150+
rootFs: rootFs,
151+
extraDir: extraDir,
152+
}
153+
}
154+
155+
func (f *fsUsageProvider) Targets() []string {
156+
return []string{f.rootFs, f.extraDir}
157+
}
158+
159+
func (f *fsUsageProvider) Usage() (*FsUsage, error) {
160+
var (
161+
rootUsage, extraUsage fs.UsageInfo
162+
rootErr, extraErr error
163+
)
164+
165+
if f.rootFs != "" {
166+
rootUsage, rootErr = f.fsInfo.GetDirUsage(f.rootFs)
167+
}
168+
169+
if f.extraDir != "" {
170+
extraUsage, extraErr = f.fsInfo.GetDirUsage(f.extraDir)
171+
}
172+
173+
usage := &FsUsage{}
174+
175+
if f.rootFs != "" && rootErr == nil {
176+
usage.InodeUsage = rootUsage.Inodes
177+
usage.BaseUsageBytes = rootUsage.Bytes
178+
usage.TotalUsageBytes = rootUsage.Bytes
179+
}
180+
181+
if f.extraDir != "" && extraErr == nil {
182+
usage.TotalUsageBytes += extraUsage.Bytes
183+
}
184+
185+
// Combine errors into a single error to return
186+
if rootErr != nil || extraErr != nil {
187+
return nil, fmt.Errorf("rootDiskErr: %v, extraDiskErr: %v", rootErr, extraErr)
188+
}
189+
190+
return usage, nil
191+
}

‎container/containerd/client.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ type ContainerdClient interface {
4545
TaskPid(ctx context.Context, id string) (uint32, error)
4646
Version(ctx context.Context) (string, error)
4747
ContainerStatus(ctx context.Context, id string) (*criapi.ContainerStatus, error)
48+
ContainerStats(ctx context.Context, id string) (*criapi.ContainerStats, error)
4849
}
4950

5051
var once sync.Once
@@ -140,6 +141,16 @@ func (c *client) ContainerStatus(ctx context.Context, id string) (*criapi.Contai
140141
return response.Status, nil
141142
}
142143

144+
func (c *client) ContainerStats(ctx context.Context, id string) (*criapi.ContainerStats, error) {
145+
response, err := c.criService.ContainerStats(ctx, &criapi.ContainerStatsRequest{
146+
ContainerId: id,
147+
})
148+
if err != nil {
149+
return nil, err
150+
}
151+
return response.Stats, nil
152+
}
153+
143154
func containerFromProto(containerpb containersapi.Container) *containers.Container {
144155
var runtime containers.RuntimeInfo
145156
if containerpb.Runtime != nil {

‎container/containerd/client_test.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
type containerdClientMock struct {
2626
cntrs map[string]*containers.Container
2727
status *criapi.ContainerStatus
28+
stats *criapi.ContainerStats
2829
returnErr error
2930
}
3031

@@ -51,10 +52,15 @@ func (c *containerdClientMock) ContainerStatus(ctx context.Context, id string) (
5152
return c.status, nil
5253
}
5354

54-
func mockcontainerdClient(cntrs map[string]*containers.Container, status *criapi.ContainerStatus, returnErr error) ContainerdClient {
55+
func (c *containerdClientMock) ContainerStats(ctx context.Context, id string) (*criapi.ContainerStats, error) {
56+
return c.stats, nil
57+
}
58+
59+
func mockcontainerdClient(cntrs map[string]*containers.Container, status *criapi.ContainerStatus, stats *criapi.ContainerStats, returnErr error) ContainerdClient {
5560
return &containerdClientMock{
5661
cntrs: cntrs,
5762
status: status,
63+
stats: stats,
5864
returnErr: returnErr,
5965
}
6066
}

‎container/containerd/factory_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ func TestCanHandleAndAccept(t *testing.T) {
5858
testContainers["40af7cdcbe507acad47a5a62025743ad3ddc6ab93b77b21363aa1c1d641047c9"] = testContainer
5959

6060
f := &containerdFactory{
61-
client: mockcontainerdClient(testContainers, nil, nil),
61+
client: mockcontainerdClient(testContainers, nil, nil, nil),
6262
cgroupSubsystems: containerlibcontainer.CgroupSubsystems{},
6363
fsInfo: nil,
6464
machineInfoFactory: nil,

‎container/containerd/handler.go

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,19 @@ import (
3232
specs "github.com/opencontainers/runtime-spec/specs-go"
3333
)
3434

35+
type fsUsageProvider struct {
36+
ctx context.Context
37+
containerId string
38+
client ContainerdClient
39+
}
40+
3541
type containerdContainerHandler struct {
3642
machineInfoFactory info.MachineInfoFactory
3743
// Absolute path to the cgroup hierarchies of this container.
3844
// (e.g.: "cpu" -> "/sys/fs/cgroup/cpu/test")
3945
cgroupPaths map[string]string
4046
fsInfo fs.FsInfo
47+
fsHandler common.FsHandler
4148
// Metadata associated with the container.
4249
reference info.ContainerReference
4350
envs map[string]string
@@ -157,6 +164,14 @@ func newContainerdContainerHandler(
157164
// Add the name and bare ID as aliases of the container.
158165
handler.image = cntr.Image
159166

167+
if includedMetrics.Has(container.DiskUsageMetrics) {
168+
handler.fsHandler = common.NewFsHandler(common.DefaultPeriod, &fsUsageProvider{
169+
ctx: ctx,
170+
client: client,
171+
containerId: id,
172+
})
173+
}
174+
160175
for _, exposedEnv := range metadataEnvAllowList {
161176
if exposedEnv == "" {
162177
// if no containerdEnvWhitelist provided, len(metadataEnvAllowList) == 1, metadataEnvAllowList[0] == ""
@@ -212,6 +227,20 @@ func (h *containerdContainerHandler) getFsStats(stats *info.ContainerStats) erro
212227
if h.includedMetrics.Has(container.DiskIOMetrics) {
213228
common.AssignDeviceNamesToDiskStats((*common.MachineInfoNamer)(mi), &stats.DiskIo)
214229
}
230+
231+
if !h.includedMetrics.Has(container.DiskUsageMetrics) {
232+
return nil
233+
}
234+
235+
// Device 、fsType and fsLimits and other information are not supported yet
236+
fsStat := info.FsStats{}
237+
usage := h.fsHandler.Usage()
238+
fsStat.BaseUsage = usage.BaseUsageBytes
239+
fsStat.Usage = usage.TotalUsageBytes
240+
fsStat.Inodes = usage.InodeUsage
241+
242+
stats.Filesystem = append(stats.Filesystem, fsStat)
243+
215244
return nil
216245
}
217246

@@ -262,12 +291,34 @@ func (h *containerdContainerHandler) Type() container.ContainerType {
262291
}
263292

264293
func (h *containerdContainerHandler) Start() {
294+
if h.fsHandler != nil {
295+
h.fsHandler.Start()
296+
}
265297
}
266298

267299
func (h *containerdContainerHandler) Cleanup() {
300+
if h.fsHandler != nil {
301+
h.fsHandler.Stop()
302+
}
268303
}
269304

270305
func (h *containerdContainerHandler) GetContainerIPAddress() string {
271306
// containerd doesnt take care of networking.So it doesnt maintain networking states
272307
return ""
273308
}
309+
310+
func (f *fsUsageProvider) Usage() (*common.FsUsage, error) {
311+
stats, err := f.client.ContainerStats(f.ctx, f.containerId)
312+
if err != nil {
313+
return nil, err
314+
}
315+
return &common.FsUsage{
316+
BaseUsageBytes: stats.WritableLayer.UsedBytes.Value,
317+
TotalUsageBytes: stats.WritableLayer.UsedBytes.Value,
318+
InodeUsage: stats.WritableLayer.InodesUsed.Value,
319+
}, nil
320+
}
321+
322+
func (f *fsUsageProvider) Targets() []string {
323+
return []string{f.containerId}
324+
}

‎container/containerd/handler_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ func TestHandler(t *testing.T) {
8888

8989
for _, ts := range []testCase{
9090
{
91-
mockcontainerdClient(nil, nil, nil),
91+
mockcontainerdClient(nil, nil, nil, nil),
9292
"/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/40af7cdcbe507acad47a5a62025743ad3ddc6ab93b77b21363aa1c1d641047c9",
9393
nil,
9494
nil,
@@ -102,7 +102,7 @@ func TestHandler(t *testing.T) {
102102
nil,
103103
},
104104
{
105-
mockcontainerdClient(testContainers, nil, nil),
105+
mockcontainerdClient(testContainers, nil, nil, nil),
106106
"/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/40af7cdcbe507acad47a5a62025743ad3ddc6ab93b77b21363aa1c1d641047c9",
107107
&mockedMachineInfo{},
108108
nil,
@@ -121,7 +121,7 @@ func TestHandler(t *testing.T) {
121121
map[string]string{},
122122
},
123123
{
124-
mockcontainerdClient(testContainers, nil, nil),
124+
mockcontainerdClient(testContainers, nil, nil, nil),
125125
"/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/40af7cdcbe507acad47a5a62025743ad3ddc6ab93b77b21363aa1c1d641047c9",
126126
&mockedMachineInfo{},
127127
nil,
@@ -140,7 +140,7 @@ func TestHandler(t *testing.T) {
140140
map[string]string{"TEST_REGION": "FRA", "TEST_ZONE": "A"},
141141
},
142142
{
143-
mockcontainerdClient(testContainers, status, nil),
143+
mockcontainerdClient(testContainers, status, nil, nil),
144144
"/kubepods/pod068e8fa0-9213-11e7-a01f-507b9d4141fa/c6a1aa99f14d3e57417e145b897e34961145f6b6f14216a176a34bfabbf79086",
145145
&mockedMachineInfo{},
146146
nil,

‎container/crio/handler.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,8 @@ func newCrioContainerHandler(
183183

184184
// we optionally collect disk usage metrics
185185
if includedMetrics.Has(container.DiskUsageMetrics) {
186-
handler.fsHandler = common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, storageLogDir, fsInfo)
186+
handler.fsHandler = common.NewFsHandler(common.DefaultPeriod, common.NewGeneralFsUsageProvider(
187+
fsInfo, rootfsStorageDir, storageLogDir))
187188
}
188189
// TODO for env vars we wanted to show from container.Config.Env from whitelist
189190
//for _, exposedEnv := range metadataEnvAllowList {

‎container/docker/handler.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,8 @@ func newDockerContainerHandler(
240240

241241
if includedMetrics.Has(container.DiskUsageMetrics) {
242242
handler.fsHandler = &dockerFsHandler{
243-
fsHandler: common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, otherStorageDir, fsInfo),
243+
fsHandler: common.NewFsHandler(common.DefaultPeriod, common.NewGeneralFsUsageProvider(
244+
fsInfo, rootfsStorageDir, otherStorageDir)),
244245
thinPoolWatcher: thinPoolWatcher,
245246
zfsWatcher: zfsWatcher,
246247
deviceID: ctnr.GraphDriver.Data["DeviceId"],

0 commit comments

Comments
 (0)