Restarting apps metrics

Author: tomez

.gitignore

@@ -35,3 +35,5 @@ marathon-forwarder
 
 # coverage
 /coverage
+
+tags

README.md

@@ -28,6 +28,31 @@ AppCop is periodically fetching applications and groups from Marathon.
 When application is suspended or group is empty for long (configurable) time then it is deleted.
 
 
+### Metrics
+
+`AppCop` provides set of standard system metrics as well as application based metrics.
+
+
+#### Metric Types
+
+`System Metrics` - `AppCop` specific telemetry (e.g - queue Size, Event delays etc). Location equals, `metrics-prefix` append `metrics-system-sub-prefix`.
+
+`Applications Metrics` - Applications telemetry calculated based on events provided by marathon
+(like: task_killed, task_finished counters). Location equals, `metrics-prefix` (append) `metrics-app-sub-prefix`.
+
+Please note the existance of `appid-prefix` config option, if set, removes matching string from
+application id when it comes to metric publication. For example, assumming
+
+```
+appid-prefix = com.example.
+appID = com.example.exampleapp
+```
+your applications metric will be placed under:
+```
+{prefix}.{metrics-app-sub-prefix}.exampleapp
+```
+
+
 ## Installation
 
 ### Installing from source code
@@ -95,6 +120,7 @@ marathon-password           |                   | Marathon password for basic au
 marathon-protocol           | `http`            | Marathon protocol (http or https)
 marathon-ssl-verify         | `true`            | Verify certificates when connecting via SSL
 marathon-timeout            | `30s`             | Time limit for requests made by the Marathon HTTP client. A timeout of zero means no timeout
+appid-prefix                |                   | Prefix common to all fully qualified application ID's. Remove this preffix from applications id's ([Metric Types](#metric types))
 marathon-username           |                   | Marathon username for basic auth
 scale-down-score            | `30`              | Score for application to scale it one instance down
 scale-limit                 | `2`               | How many scale down actions to commit in one scaling down iteration
@@ -104,6 +130,8 @@ evaluate-interval           | `30s`             | How often collected scores are
 metrics-interval            | `30s`             | Metrics reporting interval
 metrics-location            |                   | Graphite URL (used when metrics-target is set to graphite)
 metrics-prefix              | `default`         | Metrics prefix (default is resolved to <hostname>.<app_name>
+metrics-system-sub-prefix   | `appcop-internal` | System specific metrics. Append to metric-prefix
+metrics-app-sub-prefix      | `applications`    | Applications specific metrics. Appended to metric-prefix
 metrics-target              | `stdout`          | Metrics destination stdout or graphite (empty string disables metrics)
 workers-pool-size           | `10`              | Number of concurrent workers processing events
 mgc-enabled                 | `true`            | Enable garbage collecting of Marathon, old suspended applications will be deleted

config/config.go

@@ -82,6 +82,8 @@ func (config *Config) parseFlags() {
 	flag.DurationVar(&config.Marathon.Timeout,
 		"marathon-timeout", 30*time.Second,
 		"Time limit for requests made by the Marathon HTTP client. A Timeout of zero means no timeout")
+	flag.StringVar(&config.Marathon.AppIDPrefix, "appid-prefix", "",
+		"Prefix common to all fully qualified application ID's. Remove this preffix from applications id's (reffer to README to get an idea when this id is removed)")
 
 	// Score
 	flag.BoolVar(&config.Score.DryRun,
@@ -122,10 +124,16 @@ func (config *Config) parseFlags() {
 		"Metrics destination stdout or graphite (empty string disables metrics)")
 	flag.StringVar(&config.Metrics.Prefix, "metrics-prefix", "default",
 		"Metrics prefix (default is resolved to <hostname>.<app_name>")
+	flag.StringVar(&config.Metrics.SystemSubPrefix, "metrics-system-sub-prefix", "appcop-internal",
+		"System specific metrics. Append to metric-prefix")
+	flag.StringVar(&config.Metrics.AppSubPrefix, "metrics-app-sub-prefix", "applications",
+		"Applications specific metrics. Appended to metric-prefix")
 	flag.DurationVar(&config.Metrics.Interval, "metrics-interval", 30*time.Second,
 		"Metrics reporting interval")
 	flag.StringVar(&config.Metrics.Addr, "metrics-location", "",
 		"Graphite URL (used when metrics-target is set to graphite)")
+	flag.StringVar(&config.Metrics.Addr, "metrics-instance", "",
+		"Part of Graphite metric, used to distinguish between AppCop instances internal metrics.")
 
 	// Log
 	flag.StringVar(&config.Log.Level, "log-level", "info",

marathon/app.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+
+	"github.com/allegro/marathon-appcop/metrics"
 )
 
 const ApplicationImmunityLabel = "APP_IMMUNITY"
@@ -96,6 +98,28 @@ type Task struct {
 	HealthCheckResults []HealthCheckResult `json:"healthCheckResults"`
 }
 
+// GetMetric returns a string indicating where this applications metric should be placed
+// in graphite in defined prefix. It is done by triming begining prefix (if defined)
+// from application id and replacing appID separators with
+// metrics separators appropriate for graphite.
+func (t Task) GetMetric(prefix string) string {
+	taskAppID := string(t.AppID)
+
+	var appID string
+	if prefix == "" {
+		appID = taskAppID
+	} else {
+		appID = strings.Replace(taskAppID, prefix, "", 1)
+	}
+	noRootAppID := strings.TrimPrefix(appID, "/")
+	metricPath := strings.Replace(noRootAppID, metrics.PathSeparator, metrics.MetricSeparator, -1)
+	taskStatus := strings.ToLower(t.TaskStatus)
+
+	filteredPathParts := metrics.FilterOutEmptyStrings([]string{metricPath, taskStatus})
+	return strings.Join(filteredPathParts, metrics.MetricSeparator)
+
+}
+
 // TaskID from marathon
 // Usually in the form of AppID.uuid with '/' replaced with '_'
 type TaskID string

marathon/app_test.go

@@ -93,6 +93,117 @@ func TestParseTasksRecievesMalformedJSONBlob(t *testing.T) {
 	require.Error(t, err)
 }
 
+var getMetricTestCases = []struct {
+	task               *Task
+	prefix             string
+	expectedTaskMetric string
+}{
+	{
+		task: &Task{
+			TaskStatus: "task_running",
+			AppID:      "com.example.domain.context/app-name",
+		},
+		prefix:             "com.example.",
+		expectedTaskMetric: "domain.context.app-name.task_running",
+	},
+	{
+		task: &Task{
+			TaskStatus: "task_running",
+			AppID:      "/com.example.domain.context/app-name",
+		},
+		prefix:             "com.example.",
+		expectedTaskMetric: "domain.context.app-name.task_running",
+	},
+	{
+		task: &Task{
+			TaskStatus: "task_running",
+			AppID:      "com.example.domain.context/app-name",
+		},
+		prefix:             "",
+		expectedTaskMetric: "com.example.domain.context.app-name.task_running",
+	},
+	{
+		task: &Task{
+			TaskStatus: "task_running",
+			AppID:      "com.example.domain.context/group/app-name",
+		},
+		prefix:             "com.example.",
+		expectedTaskMetric: "domain.context.group.app-name.task_running",
+	},
+	{
+		task: &Task{
+			TaskStatus: "task_staging",
+			AppID:      "com.example.domain.context/group/app-name",
+		},
+		prefix:             "com.example.",
+		expectedTaskMetric: "domain.context.group.app-name.task_staging",
+	},
+	{
+		task: &Task{
+			TaskStatus: "task_staging",
+			AppID:      "com.example.domain.context/group/nested-group/app-name",
+		},
+		prefix:             "com.example.",
+		expectedTaskMetric: "domain.context.group.nested-group.app-name.task_staging",
+	},
+	{
+		task: &Task{
+			TaskStatus: "task_running",
+			AppID:      "app-name",
+		},
+		prefix:             "com.example.",
+		expectedTaskMetric: "app-name.task_running",
+	},
+	{
+		task: &Task{
+			TaskStatus: "task_running",
+			AppID:      "app-name",
+		},
+		prefix:             "",
+		expectedTaskMetric: "app-name.task_running",
+	},
+	{
+		task: &Task{
+			TaskStatus: "task_running",
+			AppID:      "com.example.domain.context/app-name",
+		},
+		prefix:             "",
+		expectedTaskMetric: "com.example.domain.context.app-name.task_running",
+	},
+	{
+		task: &Task{
+			TaskStatus: "task_running",
+			AppID:      "",
+		},
+		prefix:             "com.example.",
+		expectedTaskMetric: "task_running",
+	},
+	{
+		task: &Task{
+			TaskStatus: "",
+			AppID:      "com.example.domain.context/app-name",
+		},
+		prefix:             "com.example.",
+		expectedTaskMetric: "domain.context.app-name",
+	},
+	{
+		task: &Task{
+			TaskStatus: "",
+			AppID:      "",
+		},
+		prefix:             "com.example.",
+		expectedTaskMetric: "",
+	},
+}
+
+func TestTaskGetMetricTestCases(t *testing.T) {
+	t.Parallel()
+	for _, testCase := range getMetricTestCases {
+		taskMetric := testCase.task.GetMetric(testCase.prefix)
+		assert.Equal(t, testCase.expectedTaskMetric, taskMetric)
+	}
+}
+
 var penalizeTestCases = []struct {
 	app         *App
 	expectedApp *App

marathon/config.go

@@ -4,10 +4,15 @@ import "time"
 
 // Config contains marathon module specific configuration
 type Config struct {
-	Location  string
-	Protocol  string
-	Username  string
-	Password  string
-	VerifySsl bool
-	Timeout   time.Duration
+	Location string
+	Protocol string
+	Username string
+	Password string
+	// AppIDPrefix is a part of application id preferably present
+	// in all applications in marathon, if found it is removed for the sake of
+	// making applications paths shorter.
+	// By default this string is empty and no prefix is considered.
+	AppIDPrefix string
+	VerifySsl   bool
+	Timeout     time.Duration
 }

marathon/marathon.go

@@ -27,14 +27,16 @@ type Marathoner interface {
 	AppDelete(AppID) error
 	GroupDelete(GroupID) error
 	GetEmptyLeafGroups() ([]*Group, error)
+	GetAppIDPrefix() string
 }
 
 // Marathon reciever
 type Marathon struct {
-	Location string
-	Protocol string
-	Auth     *url.Userinfo
-	client   *pester.Client
+	Location    string
+	Protocol    string
+	appIDPrefix string
+	Auth        *url.Userinfo
+	client      *pester.Client
 }
 
 // ScaleData marathon scale json representation
@@ -81,10 +83,11 @@ func New(config Config) (*Marathon, error) {
 	pClient.Transport = transport
 
 	return &Marathon{
-		Location: config.Location,
-		Protocol: config.Protocol,
-		Auth:     auth,
-		client:   pClient,
+		Location:    config.Location,
+		Protocol:    config.Protocol,
+		appIDPrefix: config.AppIDPrefix,
+		Auth:        auth,
+		client:      pClient,
 	}, nil
 }
 
@@ -427,3 +430,7 @@ func (m Marathon) LeaderGet() (string, error) {
 
 	return leaderResponse.Leader, err
 }
+
+func (m Marathon) GetAppIDPrefix() string {
+	return m.appIDPrefix
+}

marathon/marathon_stub.go

@@ -112,3 +112,7 @@ func (m MStub) GroupDelete(groupID GroupID) error {
 func (m MStub) GetEmptyLeafGroups() ([]*Group, error) {
 	return []*Group{}, nil
 }
+
+func (m MStub) GetAppIDPrefix() string {
+	return ""
+}

metrics/config.go

@@ -4,8 +4,20 @@ import "time"
 
 // Config specific to metrics package
 type Config struct {
-	Target   string
+	Target string
+	//Prefix is the begining of metric, it is prepended
+	// in each and every published metric.
 	Prefix   string
 	Interval time.Duration
 	Addr     string
+	Instance string
+	// SystemSubPrefix it is part of a metric that is appended to the
+	// main Prefix, representing appcop internal metrics
+	// essential to appcop admins, e.g runtime metrics, event processing time,
+	// event queue size etc.
+	SystemSubPrefix string
+	// AppSubPrefix it is part of a metric that is appended to the
+	// main Prefix, representing applications specific metric, e.g task_running,
+	// task_staging, task_failed.
+	AppSubPrefix string
 }