Skip to content

Commit 0498d3f

Browse files
committed
Feature
tracing file access in layer level
1 parent 88f75de commit 0498d3f

File tree

12 files changed

+167
-69
lines changed

12 files changed

+167
-69
lines changed

errdefs/defs.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package errdefs
2+
3+
import "fmt"
4+
5+
type errWrongFormat struct {
6+
s string
7+
}
8+
9+
func (e errWrongFormat) Error() string {
10+
return fmt.Sprintf("%v has wrong format", e.s)
11+
}
12+
13+
func NewErrWrongFormat(s string) error {
14+
return errWrongFormat{s: s}
15+
}
16+
17+
func IsErrWrongFormat(err error) bool {
18+
if _, ok := err.(errWrongFormat); ok {
19+
return true
20+
}
21+
return false
22+
}

go.mod

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/YLonely/sysdig-monitor
33
go 1.12
44

55
require (
6+
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
67
github.com/Microsoft/go-winio v0.4.14 // indirect
78
github.com/docker/distribution v2.7.1+incompatible // indirect
89
github.com/docker/docker v1.13.1
@@ -16,11 +17,9 @@ require (
1617
github.com/opencontainers/image-spec v1.0.1 // indirect
1718
github.com/sirupsen/logrus v1.4.2
1819
github.com/urfave/cli v1.22.1
19-
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect
20-
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 // indirect
20+
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
2121
google.golang.org/grpc v1.25.0 // indirect
2222
gotest.tools v2.2.0+incompatible // indirect
23-
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect
2423
)
2524

2625
replace github.com/docker/docker v1.13.1 => github.com/docker/engine v0.0.0-20190822205725-ed20165a37b4

go.sum

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
2+
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
3+
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
24
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
35
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
46
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
@@ -26,6 +28,7 @@ github.com/gin-gonic/gin v1.4.0 h1:3tMoCCfM7ppqsR0ptz/wi1impNpT7/9wQtMZ8lr1mCQ=
2628
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
2729
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
2830
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
31+
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
2932
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
3033
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
3134
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -100,7 +103,10 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/p
100103
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
101104
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
102105
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
106+
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
103107
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
108+
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
109+
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
104110
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
105111
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
106112
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

server/context/context.go

Lines changed: 0 additions & 15 deletions
This file was deleted.

server/controller/container/container.go

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,8 @@ import (
99
"github.com/YLonely/sysdig-monitor/server/model"
1010
"github.com/YLonely/sysdig-monitor/server/router"
1111
"github.com/YLonely/sysdig-monitor/sysdig"
12+
"github.com/docker/docker/api/types"
1213
"github.com/docker/docker/client"
13-
"github.com/docker/docker/api/types"
14-
1514
)
1615

1716
// Container represents a top level controller for container
@@ -21,7 +20,7 @@ type ContainerController struct {
2120
ec chan sysdig.Event
2221

2322
// containers use container id as key
24-
containers map[string]*container
23+
containers map[string]*mutexContainer
2524
// container process channels
2625
containerCh map[string]chan containerEvent
2726

@@ -32,15 +31,17 @@ type ContainerController struct {
3231
dockerCli *client.Client
3332
}
3433

35-
type container struct {
36-
imageConfig *types.ImageInspect
37-
m sync.RWMutex
34+
type mutexContainer struct {
35+
m sync.RWMutex
3836
*model.Container
3937
}
4038

41-
func newContainer(id, name string, imageConfig *types.ImageInspect) *container {
42-
c := model.NewContainer(id, name)
43-
return &container{Container: c, imageConfig: imageConfig}
39+
func newMutexContainer(id, name string, containerJSON *types.ContainerJSON) (*mutexContainer, error) {
40+
c, err := model.NewContainer(id, name, containerJSON.GraphDriver.Data)
41+
if err != nil {
42+
return nil, err
43+
}
44+
return &mutexContainer{Container: c}, nil
4445
}
4546

4647
const eventBufferLen = 512
@@ -50,7 +51,7 @@ const incompleteContainerName = "incomplete"
5051
func NewController(ctx context.Context, serverErrorChannel chan<- error) (controller.Controller, error) {
5152
r := router.NewGroupRouter("/container")
5253
sysdigServer := sysdig.NewServer()
53-
res := &ContainerController{containerRouter: r, ec: sysdigServer.Subscribe(), containers: map[string]*container{}, containerCh: map[string]chan containerEvent{}}
54+
res := &ContainerController{containerRouter: r, ec: sysdigServer.Subscribe(), containers: map[string]*mutexContainer{}, containerCh: map[string]chan containerEvent{}}
5455

5556
cli, err := client.NewClientWithOpts(client.FromEnv)
5657
if err != nil {
@@ -104,12 +105,16 @@ func (cc *ContainerController) start(ctx context.Context) error {
104105
}
105106
log.L.Debug(ce)
106107
if _, exists := cc.containers[containerID]; !exists {
107-
config, err := cc.imageConfig(ctx, ce.image)
108+
containerJSON, err := cc.containerJSON(ctx, containerID)
109+
if err != nil {
110+
log.L.WithError(err).WithField("container-id", containerID).Error("cant fetch container json")
111+
continue
112+
}
113+
container, err := newMutexContainer(ce.containerID, containerName, containerJSON)
108114
if err != nil {
109-
log.L.WithError(err).WithField("container-id", containerID).Error("cant fetch image config")
115+
log.L.WithError(err).WithField("container-id", containerID).Error("")
110116
continue
111117
}
112-
container := newContainer(ce.containerID, containerName, config)
113118
ch := make(chan containerEvent, eventBufferLen)
114119
cc.cm.Lock()
115120
cc.containers[containerID] = container
@@ -136,12 +141,12 @@ func (cc *ContainerController) start(ctx context.Context) error {
136141
return nil
137142
}
138143

139-
func (cc *ContainerController) imageConfig(ctx context.Context, id string) (*types.ImageInspect, error) {
140-
imageConfig, _, err := cc.dockerCli.ImageInspectWithRaw(ctx, id)
144+
func (cc *ContainerController) containerJSON(ctx context.Context, id string) (*types.ContainerJSON, error) {
145+
j, err := cc.dockerCli.ContainerInspect(ctx, id)
141146
if err != nil {
142147
return nil, err
143148
}
144-
return &imageConfig, nil
149+
return &j, nil
145150
}
146151

147152
func convert(e sysdig.Event) containerEvent {
@@ -159,6 +164,5 @@ func convert(e sysdig.Event) containerEvent {
159164
res.rawRes = e.RawRes
160165
res.syscallType = e.SyscallType
161166
res.virtualtid = e.ThreadVirtualID
162-
res.image = e.ContainerImage
163167
return res
164168
}

server/controller/container/routes.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ type FlattenConnection struct {
2323
type GetContainerResponse struct {
2424
*model.Container
2525
ActiveConnections []FlattenConnection `json:"active_connections"`
26+
AccessedLayers []*model.LayerInfo `json:"accessed_layers"`
2627
}
2728

2829
func (cc *ContainerController) getContainer(c *gin.Context) {
@@ -40,5 +41,10 @@ func (cc *ContainerController) getContainer(c *gin.Context) {
4041
for meta, conn := range container.ActiveConnections {
4142
flattenConns = append(flattenConns, FlattenConnection{Connection: *conn, ConnectionMeta: meta})
4243
}
43-
c.JSON(200, GetContainerResponse{Container: container.Container, ActiveConnections: flattenConns})
44+
flattenLayersInfo := []*model.LayerInfo{}
45+
for _, layer := range container.LayersInOrder {
46+
flattenLayersInfo = append(flattenLayersInfo, container.AccessedLayers[layer])
47+
}
48+
49+
c.JSON(200, GetContainerResponse{Container: container.Container, ActiveConnections: flattenConns, AccessedLayers: flattenLayersInfo})
4450
}

server/controller/container/util.go

Lines changed: 49 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ import (
44
"context"
55
"errors"
66
"fmt"
7+
"os"
78
"strconv"
89
"strings"
910
"time"
1011

12+
"github.com/YLonely/sysdig-monitor/errdefs"
1113
"github.com/YLonely/sysdig-monitor/log"
1214
"github.com/YLonely/sysdig-monitor/server/model"
1315
)
@@ -29,10 +31,9 @@ type containerEvent struct {
2931
rawRes int
3032
syscallType string
3133
virtualtid int
32-
image string
3334
}
3435

35-
func processLoop(ctx context.Context, c *container, ch chan containerEvent) error {
36+
func processLoop(ctx context.Context, c *mutexContainer, ch chan containerEvent) error {
3637
var e containerEvent
3738
var err error
3839
for {
@@ -64,8 +65,8 @@ func processLoop(ctx context.Context, c *container, ch chan containerEvent) erro
6465
}
6566
} else {
6667
// may have some other handler?
67-
if err = handleNetwork(c.Container, e); err != nil {
68-
//log.L.WithField("container-id", c.ID).WithError(err).Error("network handler error")
68+
if err = handleNetwork(c.Container, e); err != nil && !errdefs.IsErrWrongFormat(err) {
69+
log.L.WithField("container-id", c.ID).WithError(err).Warning("network handler error")
6970
}
7071
}
7172
c.m.Unlock()
@@ -79,10 +80,14 @@ func handleSysCall(c *model.Container, e containerEvent) error {
7980
if len(syscall) <= 0 {
8081
return nil
8182
}
82-
if _, exists := c.IndividualCalls[syscall]; !exists {
83-
c.IndividualCalls[syscall] = &model.SystemCall{Name: syscall}
83+
var (
84+
call *model.SystemCall
85+
exists bool
86+
)
87+
if call, exists = c.IndividualCalls[syscall]; !exists {
88+
call = &model.SystemCall{Name: syscall}
89+
c.IndividualCalls[syscall] = call
8490
}
85-
call := c.IndividualCalls[syscall]
8691
call.Calls++
8792
call.TotalTime += latency
8893
c.SystemCalls.TotalCalls++
@@ -106,10 +111,14 @@ func handleNetIO(c *model.Container, e containerEvent) error {
106111
}
107112
// if event shows that a net io begins before "connect" or "accpet",
108113
// we just ignore the error sequence and add a new connection
109-
if _, exists := c.ActiveConnections[meta]; !exists {
110-
c.ActiveConnections[meta] = &model.Connection{Type: e.fdType}
114+
var (
115+
conn *model.Connection
116+
exists bool
117+
)
118+
if conn, exists = c.ActiveConnections[meta]; !exists {
119+
conn = &model.Connection{Type: e.fdType}
120+
c.ActiveConnections[meta] = conn
111121
}
112-
conn := c.ActiveConnections[meta]
113122
if e.isIORead {
114123
conn.ReadIn += int64(bufLen)
115124
c.Network.TotalReadIn += int64(bufLen)
@@ -123,14 +132,25 @@ func handleNetIO(c *model.Container, e containerEvent) error {
123132
func handleFileIO(c *model.Container, e containerEvent) error {
124133
fileName := e.fdName
125134
bufLen := e.bufferLen
126-
if _, exists := c.AccessedFiles[fileName]; !exists {
127-
c.AccessedFiles[fileName] = &model.File{Name: fileName}
135+
var (
136+
file *model.File
137+
exists bool
138+
)
139+
140+
if file, exists = c.AccessedFiles[fileName]; !exists {
141+
file = &model.File{Name: fileName}
142+
err := attachToLayer(c, file)
143+
if err != nil {
144+
return nil
145+
}
146+
c.AccessedFiles[fileName] = file
128147
}
129-
file := c.AccessedFiles[fileName]
130148
if e.isIOWrite {
131149
file.WriteOut += int64(bufLen)
150+
file.Layer.WriteOut += int64(bufLen)
132151
c.FileSystem.TotalWriteOut += int64(bufLen)
133152
} else if e.isIORead {
153+
file.Layer.ReadIn += int64(bufLen)
134154
file.ReadIn += int64(bufLen)
135155
c.FileSystem.TotalReadIn += int64(bufLen)
136156
}
@@ -179,7 +199,7 @@ func connectionMeta(fdname string) (model.ConnectionMeta, error) {
179199
parts := strings.Split(fdname, "->")
180200
meta := model.ConnectionMeta{}
181201
if len(parts) != 2 {
182-
return meta, fmt.Errorf("wrong connection meta format:%v", fdname)
202+
return meta, errdefs.NewErrWrongFormat(fdname)
183203
}
184204
source, dest := parts[0], parts[1]
185205

@@ -199,15 +219,27 @@ func splitAddress(address string) (string, int, error) {
199219
err error
200220
)
201221
if len(address) <= 1 {
202-
return "", -1, errors.New("empty address")
222+
return "", -1, errdefs.NewErrWrongFormat(address)
203223
}
204224
for portStart = len(address) - 1; portStart >= 0 && address[portStart] != ':'; portStart-- {
205225
}
206226
if portStart <= 0 {
207-
return "", -1, errors.New("no port address")
227+
return "", -1, errdefs.NewErrWrongFormat(address)
208228
}
209229
if port, err = strconv.Atoi(address[portStart+1:]); err != nil {
210-
return "", -1, fmt.Errorf("wrong address format:%v", address)
230+
return "", -1, errdefs.NewErrWrongFormat(address)
211231
}
212232
return address[:portStart], port, nil
213233
}
234+
235+
func attachToLayer(c *model.Container, file *model.File) error {
236+
fileName := file.Name
237+
for _, dir := range c.LayersInOrder {
238+
if _, err := os.Stat(dir + fileName); err == nil {
239+
c.AccessedLayers[dir].AccessedFiles[fileName] = file
240+
file.Layer = c.AccessedLayers[dir]
241+
return nil
242+
}
243+
}
244+
return fmt.Errorf("cant find file %v in any of lower layers", fileName)
245+
}

0 commit comments

Comments
 (0)