Feature & Refactoring

1. Remove localprom package (update behavior of the metrics cant be separated from the updating of container status)
2. Add some prometheus metrics of container (export at /metrics)

Author: YLonely

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/prometheus/client_golang v1.2.1 // indirect
+	github.com/prometheus/client_golang v1.2.1
 	github.com/sirupsen/logrus v1.4.2
 	github.com/urfave/cli v1.22.1
 	golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect

go.sum

@@ -54,12 +54,14 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -134,6 +136,7 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -150,6 +153,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/p
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

localprom/localprom.go

@@ -12,7 +12,6 @@ import (
 	"github.com/YLonely/sysdig-monitor/sysdig"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/client"
-
 )
 
 type containerController struct {

localprom/task.go

@@ -0,0 +1,98 @@
+package container
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const (
+	metricsNamespace = "sysdig_monitor"
+	metricsSubsystem = "container"
+
+	containerIDLabel            = "id"
+	syscallTypeLabel            = "syscall_type"
+	containerLayerDirLabel      = "layer_dir"
+	containerFileLabel          = "file_name"
+	containerConnectionSrcIP    = "src_ip"
+	containerConnectionDestIP   = "dest_ip"
+	containerConnectionSrcPort  = "src_port"
+	containerConnectionDestPort = "dest_port"
+)
+
+func newGaugeOpts(name, help string) prometheus.GaugeOpts {
+	return prometheus.GaugeOpts{Subsystem: metricsSubsystem, Namespace: metricsNamespace, Name: name, Help: help}
+}
+
+var (
+	// containerIDMetric is a metric tracks the container's name
+	containerIDMetric = promauto.NewGaugeVec(
+		newGaugeOpts(
+			"container_id",
+			"The container's id, the value is always 1",
+		),
+		[]string{containerIDLabel},
+	)
+
+	//systemCallCount tracks the total invoke times of different types of system call for every container
+	systemCallCount = promauto.NewGaugeVec(
+		newGaugeOpts(
+			"syscall_total",
+			"Total invoke times of different system call",
+		),
+		[]string{containerIDLabel, syscallTypeLabel},
+	)
+
+	//systemCallTotalLatency tracks the total latency of different system call in seconds for every container
+	systemCallTotalLatency = promauto.NewGaugeVec(
+		newGaugeOpts(
+			"system_call_total_latency_seconds",
+			"Total latency of the system calls of container",
+		),
+		[]string{containerIDLabel, syscallTypeLabel},
+	)
+
+	// containerLayerDir is metric tracks the layer dir of each container
+	containerLayerDir = promauto.NewGaugeVec(
+		newGaugeOpts(
+			"layer_dir",
+			"The dir of the layer of each container, the upper layer's value is 1, the deeper the bigger",
+		),
+		[]string{containerIDLabel, containerLayerDirLabel},
+	)
+
+	// containerLayerFileRead tracks the total bytes a container read from a file which belongs to a layer
+	containerLayerFileRead = promauto.NewGaugeVec(
+		newGaugeOpts(
+			"layer_file_read_bytes",
+			"The total bytes a container read from a file which belongs to a layer",
+		),
+		[]string{containerIDLabel, containerLayerDirLabel, containerFileLabel},
+	)
+
+	// containerLayerFileWrite tracks the total bytes a container write to a file which belongs to a layer
+	containerLayerFileWrite = promauto.NewGaugeVec(
+		newGaugeOpts(
+			"layer_file_write_bytes",
+			"The total bytes a container write from a file which belongs to a layer",
+		),
+		[]string{containerIDLabel, containerLayerDirLabel, containerFileLabel},
+	)
+
+	// containerActiveConnectionRead tracks the total bytes a container read from a active net connection
+	containerActiveConnectionRead = promauto.NewGaugeVec(
+		newGaugeOpts(
+			"active_connection_read_bytes",
+			"The total bytes a container read from a active net connection",
+		),
+		[]string{containerIDLabel, containerConnectionSrcIP, containerConnectionSrcPort, containerConnectionDestIP, containerConnectionDestPort},
+	)
+
+	// containerActiveConnectionWrite tracks the total bytes a container write to a active new connection
+	containerActiveConnectionWrite = promauto.NewGaugeVec(
+		newGaugeOpts(
+			"active_connection_write_bytes",
+			"The total bytes a container write to a active net connection",
+		),
+		[]string{containerIDLabel, containerConnectionSrcIP, containerConnectionSrcPort, containerConnectionDestIP, containerConnectionDestPort},
+	)
+)

server/controller/container/container.go

@@ -36,6 +36,12 @@ type containerEvent struct {
 func processLoop(ctx context.Context, c *mutexContainer, ch chan containerEvent) error {
 	var e containerEvent
 	var err error
+	containerIDMetric.WithLabelValues(c.ID).Set(1)
+	depth := 1
+	for _, l := range c.LayersInOrder {
+		containerLayerDir.WithLabelValues(c.ID, l).Set(float64(depth))
+		depth++
+	}
 	for {
 		select {
 		case e = <-ch:
@@ -48,6 +54,7 @@ func processLoop(ctx context.Context, c *mutexContainer, ch chan containerEvent)
 		// container exits
 		if e.eventType == "procexit" && e.virtualtid == 1 {
 			log.L.WithField("container-id", c.ID).Debug("container exits")
+			cleanUpMetrics(c.Container)
 			return nil
 		}
 		if e.eventDir == "<" {
@@ -91,6 +98,8 @@ func handleSysCall(c *model.Container, e containerEvent) error {
 	call.Calls++
 	call.TotalTime += latency
 	c.SystemCalls.TotalCalls++
+	systemCallCount.WithLabelValues(c.ID, syscall).Set(float64(call.Calls))
+	systemCallTotalLatency.WithLabelValues(c.ID, syscall).Set(float64(call.TotalTime) / float64(time.Second))
 	return nil
 }
 
@@ -119,12 +128,15 @@ func handleNetIO(c *model.Container, e containerEvent) error {
 		conn = &model.Connection{Type: e.fdType}
 		c.ActiveConnections[meta] = conn
 	}
+	sip, sport, dip, dport := meta.SourceIP, strconv.Itoa(meta.SourcePort), meta.DestIP, strconv.Itoa(meta.DestPort)
 	if e.isIORead {
 		conn.ReadIn += int64(bufLen)
 		c.Network.TotalReadIn += int64(bufLen)
+		containerActiveConnectionRead.WithLabelValues(c.ID, sip, sport, dip, dport).Set(float64(conn.ReadIn))
 	} else if e.isIOWrite {
 		conn.WriteOut += int64(bufLen)
 		c.Network.TotalWriteOut += int64(bufLen)
+		containerActiveConnectionWrite.WithLabelValues(c.ID, sip, sport, dip, dport).Set(float64(conn.WriteOut))
 	}
 	return nil
 }
@@ -145,14 +157,17 @@ func handleFileIO(c *model.Container, e containerEvent) error {
 		}
 		c.AccessedFiles[fileName] = file
 	}
+	layerDir := file.Layer.Dir
 	if e.isIOWrite {
 		file.WriteOut += int64(bufLen)
 		file.Layer.WriteOut += int64(bufLen)
 		c.FileSystem.TotalWriteOut += int64(bufLen)
+		containerLayerFileWrite.WithLabelValues(c.ID, layerDir, fileName).Set(float64(file.WriteOut))
 	} else if e.isIORead {
 		file.Layer.ReadIn += int64(bufLen)
 		file.ReadIn += int64(bufLen)
 		c.FileSystem.TotalReadIn += int64(bufLen)
+		containerLayerFileRead.WithLabelValues(c.ID, layerDir, fileName).Set(float64(file.ReadIn))
 	}
 	latency := e.latency
 	latency /= time.Millisecond
@@ -188,8 +203,11 @@ func handleNetwork(c *model.Container, e containerEvent) error {
 			return err
 		}
 		// should return nonexists error?
+		sip, sport, dip, dport := meta.SourceIP, strconv.Itoa(meta.SourcePort), meta.DestIP, strconv.Itoa(meta.DestPort)
 		if _, exists := c.ActiveConnections[meta]; exists {
 			delete(c.ActiveConnections, meta)
+			containerActiveConnectionRead.DeleteLabelValues(c.ID, sip, sport, dip, dport)
+			containerActiveConnectionWrite.DeleteLabelValues(c.ID, sip, sport, dip, dport)
 		}
 	}
 	return nil
@@ -243,3 +261,30 @@ func attachToLayer(c *model.Container, file *model.File) error {
 	}
 	return fmt.Errorf("cant find file %v in any of lower layers", fileName)
 }
+
+func cleanUpMetrics(container *model.Container) {
+	// clean up connection metrics
+	for meta, _ := range container.ActiveConnections {
+		sip, sport, dip, dport := meta.SourceIP, strconv.Itoa(meta.SourcePort), meta.DestIP, strconv.Itoa(meta.DestPort)
+		containerActiveConnectionRead.DeleteLabelValues(container.ID, sip, sport, dip, dport)
+		containerActiveConnectionWrite.DeleteLabelValues(container.ID, sip, sport, dip, dport)
+	}
+
+	// clean up layer file metrics and layer metric
+	for layerDir, layerInfo := range container.AccessedLayers {
+		for fileName, _ := range layerInfo.AccessedFiles {
+			containerLayerFileRead.DeleteLabelValues(container.ID, layerDir, fileName)
+			containerLayerFileWrite.DeleteLabelValues(container.ID, layerDir, fileName)
+		}
+		containerLayerDir.DeleteLabelValues(container.ID, layerDir)
+	}
+
+	// clean up system call count and latency metrics
+	for syscall, _ := range container.IndividualCalls {
+		systemCallCount.DeleteLabelValues(container.ID, syscall)
+		systemCallTotalLatency.DeleteLabelValues(container.ID, syscall)
+	}
+
+	// delete this container
+	containerIDMetric.DeleteLabelValues(container.ID)
+}

server/controller/container/metrics.go

@@ -1,11 +1,9 @@
 package prometheus
 
 import (
-	"github.com/YLonely/sysdig-monitor/localprom"
 	"github.com/gin-gonic/gin"
 )
 
 func (pc *prometheusContorller) metrics(c *gin.Context) {
-	localprom.RunMetrics()
 	pc.handler.ServeHTTP(c.Writer, c.Request)
 }

server/controller/container/prometheus.go

@@ -54,7 +54,7 @@ func (s *server) Start(ctx context.Context) chan error {
 			errch <- err
 		}
 	}()
-	log.L.Info("web server start")
+	log.L.Info("web server start. Listening on port " + s.conf.Port)
 	return errch
 }