Adding support for custom metricDimensions.json for statsd metrics emitter (#250)

* Adding support for custom metricDimensions.json.

* Addressing Adheip's comments.

* Fixing test failure.

Author: harinirajendran

apis/druid/v1alpha1/druid_types.go

@@ -143,6 +143,9 @@ type DruidSpec struct {
 	Zookeeper     *ZookeeperSpec     `json:"zookeeper,omitempty"`
 	MetadataStore *MetadataStoreSpec `json:"metadataStore,omitempty"`
 	DeepStorage   *DeepStorageSpec   `json:"deepStorage,omitempty"`
+
+	// Optional: Custom Dimension Map Path for statsd emitter
+	DimensionsMapPath string `json:"metricDimensions.json,omitempty"`
 }
 
 type DruidNodeSpec struct {

chart/templates/crds/druid.apache.org_druids.yaml

@@ -641,6 +641,8 @@ spec:
                 - spec
                 - type
                 type: object
+              metricDimensions.json:
+                type: string
               nodeSelector:
                 additionalProperties:
                   type: string

controllers/druid/handler.go

@@ -954,11 +954,19 @@ func makeCommonConfigMap(m *v1alpha1.Druid, ls map[string]string) (*v1.ConfigMap
 		}
 	}
 
+	data := map[string]string{
+		"common.runtime.properties": prop,
+	}
+
+	if m.Spec.DimensionsMapPath != "" {
+		data["metricDimensions.json"] = m.Spec.DimensionsMapPath
+	}
+
 	cfg, err := makeConfigMap(
 		fmt.Sprintf("%s-druid-common-config", m.ObjectMeta.Name),
 		m.Namespace,
 		ls,
-		map[string]string{"common.runtime.properties": prop})
+		data)
 	return cfg, err
 }
 

controllers/druid/testdata/common-config-map.yaml

@@ -52,6 +52,10 @@ data:
     druid.storage.baseKey=druid/segments
     druid.s3.accessKey=accesskey
     druid.s3.secretKey=secretkey
+  metricDimensions.json: |-
+    {
+      "query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer"}
+    }
 kind: ConfigMap
 metadata:
   labels:
@@ -60,4 +64,4 @@ metadata:
   name: druid-test-druid-common-config
   namespace: test-namespace
   annotations:
-    druidOpResourceHash: +30qwQo2Inbc9o49vF664RAPMMQ=
+    druidOpResourceHash: 1/BkoZEvfj+rB80LzeQeu6T7mhs=

controllers/druid/testdata/druid-smoke-test-cluster.yaml

@@ -82,6 +82,10 @@ spec:
     druid.indexer.logs.type=file
     druid.indexer.logs.directory=/druid/data/indexing-logs
     druid.lookup.enableLookupSyncOnStartup=false
+  metricDimensions.json: |-
+    {
+      "query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer"}
+    }
   volumeMounts:
     - mountPath: /druid/data
       name: data-volume

controllers/druid/testdata/druid-test-cr.yaml

@@ -100,6 +100,10 @@ spec:
       # Storage type of double columns
       # ommiting this will lead to index double as float at the storage layer
       druid.indexing.doubleStorage=double
+  metricDimensions.json: |-
+    {
+      "query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer"}
+    }
   nodes:
     brokers:
       nodeType: "broker"

deploy/crds/druid.apache.org_druids.yaml

@@ -641,6 +641,8 @@ spec:
                 - spec
                 - type
                 type: object
+              metricDimensions.json:
+                type: string
               nodeSelector:
                 additionalProperties:
                   type: string

examples/tiny-cluster.yaml

@@ -82,6 +82,170 @@ spec:
     druid.indexer.logs.type=file
     druid.indexer.logs.directory=/druid/data/indexing-logs
     druid.lookup.enableLookupSyncOnStartup=false
+
+  metricDimensions.json: |-
+    {
+      "query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer"},
+      "query/bytes" : { "dimensions" : ["dataSource", "type"], "type" : "count"},
+      "query/node/time" : { "dimensions" : ["server"], "type" : "timer"},
+      "query/node/ttfb" : { "dimensions" : ["server"], "type" : "timer"},
+      "query/node/bytes" : { "dimensions" : ["server"], "type" : "count"},
+      "query/node/backpressure": { "dimensions" : ["server"], "type" : "timer"},
+      "query/intervalChunk/time" : { "dimensions" : [], "type" : "timer"},
+
+      "query/segment/time" : { "dimensions" : [], "type" : "timer"},
+      "query/wait/time" : { "dimensions" : [], "type" : "timer"},
+      "segment/scan/pending" : { "dimensions" : [], "type" : "gauge"},
+      "query/segmentAndCache/time" : { "dimensions" : [], "type" : "timer" },
+      "query/cpu/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer" },
+
+      "query/count" : { "dimensions" : [], "type" : "count" },
+      "query/success/count" : { "dimensions" : [], "type" : "count" },
+      "query/failed/count" : { "dimensions" : [], "type" : "count" },
+      "query/interrupted/count" : { "dimensions" : [], "type" : "count" },
+      "query/timeout/count" : { "dimensions" : [], "type" : "count" },
+
+      "query/cache/delta/numEntries" : { "dimensions" : [], "type" : "count" },
+      "query/cache/delta/sizeBytes" : { "dimensions" : [], "type" : "count" },
+      "query/cache/delta/hits" : { "dimensions" : [], "type" : "count" },
+      "query/cache/delta/misses" : { "dimensions" : [], "type" : "count" },
+      "query/cache/delta/evictions" : { "dimensions" : [], "type" : "count" },
+      "query/cache/delta/hitRate" : { "dimensions" : [], "type" : "count", "convertRange" : true },
+      "query/cache/delta/averageBytes" : { "dimensions" : [], "type" : "count" },
+      "query/cache/delta/timeouts" : { "dimensions" : [], "type" : "count" },
+      "query/cache/delta/errors" : { "dimensions" : [], "type" : "count" },
+
+      "query/cache/total/numEntries" : { "dimensions" : [], "type" : "gauge" },
+      "query/cache/total/sizeBytes" : { "dimensions" : [], "type" : "gauge" },
+      "query/cache/total/hits" : { "dimensions" : [], "type" : "gauge" },
+      "query/cache/total/misses" : { "dimensions" : [], "type" : "gauge" },
+      "query/cache/total/evictions" : { "dimensions" : [], "type" : "gauge" },
+      "query/cache/total/hitRate" : { "dimensions" : [], "type" : "gauge", "convertRange" : true },
+      "query/cache/total/averageBytes" : { "dimensions" : [], "type" : "gauge" },
+      "query/cache/total/timeouts" : { "dimensions" : [], "type" : "gauge" },
+      "query/cache/total/errors" : { "dimensions" : [], "type" : "gauge" },
+
+      "ingest/events/thrownAway" : { "dimensions" : ["dataSource"], "type" : "count" },
+      "ingest/events/unparseable" : { "dimensions" : ["dataSource"], "type" : "count" },
+      "ingest/events/duplicate" : { "dimensions" : ["dataSource"], "type" : "count" },
+      "ingest/events/processed" : { "dimensions" : ["dataSource", "taskType", "taskId"], "type" : "count" },
+      "ingest/events/messageGap" : { "dimensions" : ["dataSource"], "type" : "gauge" },
+      "ingest/rows/output" : { "dimensions" : ["dataSource"], "type" : "count" },
+      "ingest/persists/count" : { "dimensions" : ["dataSource"], "type" : "count" },
+      "ingest/persists/time" : { "dimensions" : ["dataSource"], "type" : "timer" },
+      "ingest/persists/cpu" : { "dimensions" : ["dataSource"], "type" : "timer" },
+      "ingest/persists/backPressure" : { "dimensions" : ["dataSource"], "type" : "gauge" },
+      "ingest/persists/failed" : { "dimensions" : ["dataSource"], "type" : "count" },
+      "ingest/handoff/failed" : { "dimensions" : ["dataSource"], "type" : "count" },
+      "ingest/merge/time" : { "dimensions" : ["dataSource"], "type" : "timer" },
+      "ingest/merge/cpu" : { "dimensions" : ["dataSource"], "type" : "timer" },
+
+      "ingest/kafka/lag" : { "dimensions" : ["dataSource"], "type" : "gauge" },
+      "ingest/kafka/maxLag" : { "dimensions" : ["dataSource"], "type" : "gauge" },
+      "ingest/kafka/avgLag" : { "dimensions" : ["dataSource"], "type" : "gauge" },
+
+      "task/success/count" : { "dimensions" : ["dataSource"], "type" : "count" },
+      "task/failed/count" : { "dimensions" : ["dataSource"], "type" : "count" },
+      "task/running/count" : { "dimensions" : ["dataSource"], "type" : "gauge" },
+      "task/pending/count" : { "dimensions" : ["dataSource"], "type" : "gauge" },
+      "task/waiting/count" : { "dimensions" : ["dataSource"], "type" : "gauge" },
+
+      "taskSlot/total/count" : { "dimensions" : [], "type" : "gauge" },
+      "taskSlot/idle/count" : { "dimensions" : [], "type" : "gauge" },
+      "taskSlot/busy/count" : { "dimensions" : [], "type" : "gauge" },
+      "taskSlot/lazy/count" : { "dimensions" : [], "type" : "gauge" },
+      "taskSlot/blacklisted/count" : { "dimensions" : [], "type" : "gauge" },
+
+      "task/run/time" : { "dimensions" : ["dataSource", "taskType"], "type" : "timer" },
+      "segment/added/bytes" : { "dimensions" : ["dataSource", "taskType"], "type" : "count" },
+      "segment/moved/bytes" : { "dimensions" : ["dataSource", "taskType"], "type" : "count" },
+      "segment/nuked/bytes" : { "dimensions" : ["dataSource", "taskType"], "type" : "count" },
+
+      "segment/assigned/count" : { "dimensions" : ["tier"], "type" : "count" },
+      "segment/moved/count" : { "dimensions" : ["tier"], "type" : "count" },
+      "segment/dropped/count" : { "dimensions" : ["tier"], "type" : "count" },
+      "segment/deleted/count" : { "dimensions" : ["tier"], "type" : "count" },
+      "segment/unneeded/count" : { "dimensions" : ["tier"], "type" : "count" },
+      "segment/unavailable/count" : { "dimensions" : ["dataSource"], "type" : "gauge" },
+      "segment/underReplicated/count" : { "dimensions" : ["dataSource", "tier"], "type" : "gauge" },
+      "segment/cost/raw" : { "dimensions" : ["tier"], "type" : "count" },
+      "segment/cost/normalization" : { "dimensions" : ["tier"], "type" : "count" },
+      "segment/cost/normalized" : { "dimensions" : ["tier"], "type" : "count" },
+      "segment/loadQueue/size" : { "dimensions" : ["server"], "type" : "gauge" },
+      "segment/loadQueue/failed" : { "dimensions" : ["server"], "type" : "gauge" },
+      "segment/loadQueue/count" : { "dimensions" : ["server"], "type" : "gauge" },
+      "segment/dropQueue/count" : { "dimensions" : ["server"], "type" : "gauge" },
+      "segment/size" : { "dimensions" : ["dataSource"], "type" : "gauge" },
+      "segment/overShadowed/count" : { "dimensions" : [], "type" : "gauge" },
+
+      "segment/max" : { "dimensions" : [], "type" : "gauge"},
+      "segment/used" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" },
+      "segment/usedPercent" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge", "convertRange" : true },
+      "segment/pendingDelete" : { "dimensions" : [], "type" : "gauge"},
+
+      "jvm/pool/committed" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" },
+      "jvm/pool/init" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" },
+      "jvm/pool/max" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" },
+      "jvm/pool/used" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" },
+      "jvm/bufferpool/count" : { "dimensions" : ["bufferpoolName"], "type" : "gauge" },
+      "jvm/bufferpool/used" : { "dimensions" : ["bufferpoolName"], "type" : "gauge" },
+      "jvm/bufferpool/capacity" : { "dimensions" : ["bufferpoolName"], "type" : "gauge" },
+      "jvm/mem/init" : { "dimensions" : ["memKind"], "type" : "gauge" },
+      "jvm/mem/max" : { "dimensions" : ["memKind"], "type" : "gauge" },
+      "jvm/mem/used" : { "dimensions" : ["memKind"], "type" : "gauge" },
+      "jvm/mem/committed" : { "dimensions" : ["memKind"], "type" : "gauge" },
+      "jvm/gc/count" : { "dimensions" : ["gcName", "gcGen"], "type" : "count" },
+      "jvm/gc/cpu" : { "dimensions" : ["gcName", "gcGen"], "type" : "count" },
+
+      "ingest/events/buffered" : { "dimensions" : ["serviceName", "bufferCapacity"], "type" : "gauge"},
+
+      "sys/swap/free" : { "dimensions" : [], "type" : "gauge"},
+      "sys/swap/max" : { "dimensions" : [], "type" : "gauge"},
+      "sys/swap/pageIn" : { "dimensions" : [], "type" : "gauge"},
+      "sys/swap/pageOut" : { "dimensions" : [], "type" : "gauge"},
+      "sys/disk/write/count" : { "dimensions" : ["fsDevName"], "type" : "count"},
+      "sys/disk/read/count" : { "dimensions" : ["fsDevName"], "type" : "count"},
+      "sys/disk/write/size" : { "dimensions" : ["fsDevName"], "type" : "count"},
+      "sys/disk/read/size" : { "dimensions" : ["fsDevName"], "type" : "count"},
+      "sys/net/write/size" : { "dimensions" : [], "type" : "count"},
+      "sys/net/read/size" : { "dimensions" : [], "type" : "count"},
+      "sys/fs/used" : { "dimensions" : ["fsDevName", "fsDirName", "fsTypeName", "fsSysTypeName", "fsOptions"], "type" : "gauge"},
+      "sys/fs/max" : { "dimensions" : ["fsDevName", "fsDirName", "fsTypeName", "fsSysTypeName", "fsOptions"], "type" : "gauge"},
+      "sys/mem/used" : { "dimensions" : [], "type" : "gauge"},
+      "sys/mem/max" : { "dimensions" : [], "type" : "gauge"},
+      "sys/storage/used" : { "dimensions" : ["fsDirName"], "type" : "gauge"},
+      "sys/cpu" : { "dimensions" : ["cpuName", "cpuTime"], "type" : "gauge"},
+
+      "coordinator-segment/count" : { "dimensions" : ["dataSource"], "type" : "gauge" },
+      "historical-segment/count" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" },
+
+      "jetty/numOpenConnections" : { "dimensions" : [], "type" : "gauge" },
+      "query/cache/caffeine/total/requests" : { "dimensions" : [], "type" : "gauge" },
+      "query/cache/caffeine/total/loadTime" : { "dimensions" : [], "type" : "gauge" },
+      "query/cache/caffeine/total/evictionBytes" : { "dimensions" : [], "type" : "gauge" },
+      "query/cache/memcached/total" : { "dimensions" : ["[MEM] Reconnecting Nodes (ReconnectQueue)",
+        "[MEM] Request Rate: All",
+        "[MEM] Average Bytes written to OS per write",
+        "[MEM] Average Bytes read from OS per read",
+        "[MEM] Response Rate: All (Failure + Success + Retry)",
+        "[MEM] Response Rate: Retry",
+        "[MEM] Response Rate: Failure",
+        "[MEM] Response Rate: Success"],
+        "type" : "gauge" },
+      "query/cache/caffeine/delta/requests" : { "dimensions" : [], "type" : "count" },
+      "query/cache/caffeine/delta/loadTime" : { "dimensions" : [], "type" : "count" },
+      "query/cache/caffeine/delta/evictionBytes" : { "dimensions" : [], "type" : "count" },
+      "query/cache/memcached/delta" : { "dimensions" : ["[MEM] Reconnecting Nodes (ReconnectQueue)",
+        "[MEM] Request Rate: All",
+        "[MEM] Average Bytes written to OS per write",
+        "[MEM] Average Bytes read from OS per read",
+        "[MEM] Response Rate: All (Failure + Success + Retry)",
+        "[MEM] Response Rate: Retry",
+        "[MEM] Response Rate: Failure",
+        "[MEM] Response Rate: Success"],
+        "type" : "count" }
+    }
+
   volumeMounts:
     - mountPath: /druid/data
       name: data-volume