Add enableLokiLogs to commonlib signals and new log signal type (#1399)

* Add enableLokiLogs to commonlib signals

* Update common in logs

* Update changelog

* whitespace fmt

Author: v-zhuravlev

common-lib/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 0.3.0
+
+- [Signal] add new signal `log`.
+- [Signal] add enableLokiLogs=true|false to signals init.
+- [Signal] `withExprWrappersMixin(offset)` - wrap signal expression into additional function on top of existing wrappers.
+
+
 # 0.2.0
 
 - [Signal] `withOffset(offset)` - add offset modifier to the expression.

common-lib/common/signal/README.md

@@ -82,6 +82,7 @@ Init level:
 |rangeFunction| Rate function to use for counter metrics.|rate,irate,delta,idelta,increase|`rate`|`increase`|
 |varAdHocEnabled| Attach ad hoc labels to variables generated. |`true`,`false`|`false`|`false`|
 |varAdHocLabels| Limit ad hoc to the specific labels |*|`["environment"]`|`[]`|
+|enableLokiLogs| Add additional loki datasource to variables generation |`true`,`false`|`true`|`false`|
 
 Signal's level:
 

common-lib/common/signal/base.libsonnet

@@ -18,7 +18,6 @@ local xtd = import 'github.com/jsonnet-libs/xtd/main.libsonnet';
   ): {
 
     local prometheusQuery = g.query.prometheus,
-    local lokiQuery = g.query.loki,
     local this = self,
     local hasValueMaps = std.length(this.getValueMappings(this.sourceMaps)) > 0,
     local legendCustomTemplate = sourceMaps[0].legendCustomTemplate,

common-lib/common/signal/log.libsonnet

@@ -0,0 +1,68 @@
+local g = import '../g.libsonnet';
+local utils = import '../utils.libsonnet';
+local base = import './base.libsonnet';
+local signalUtils = import './utils.libsonnet';
+local lokiQuery = g.query.loki;
+base {
+  new(
+    name,
+    type,
+    unit,
+    nameShort,
+    description,
+    aggLevel,
+    aggFunction,
+    vars,
+    datasource,
+    sourceMaps,
+  ):
+    base.new(
+      name,
+      type,
+      unit,
+      nameShort,
+      description,
+      aggLevel,
+      aggFunction,
+      vars,
+      datasource,
+      sourceMaps=sourceMaps,
+    )
+    {
+      common(type)::
+        // override panel-wide --mixed-- datasource
+        lokiQuery.withDatasource('${%s}' % datasource)
+        + g.panel.logs.panelOptions.withDescription(description),
+
+
+      //Return as grafana panel target(query+legend)
+      asTarget(name=name):
+        lokiQuery.new(
+          '${%s}' % datasource,
+          self.asPanelExpression(),
+        )
+        + lokiQuery.withRefId(name),
+
+      asGauge()::
+        error 'asGauge() is not supported for log metrics.',
+      asStat()::
+        error 'asStat() is not supported for log metrics.',
+      asTimeSeries()::
+        error 'asTimeSeries() is not supported for log metrics.',
+      asTableTarget():
+        error 'asTableTarget() is not supported for log metrics.',
+      asTableColumn():
+        error 'asTableColumn() is not supported for log metrics.',
+      asTable():
+        error 'asTable() is not supported for log metrics.',
+      asOverride():
+        error 'asOverride() is not supported for log metrics.',
+      asStatusHistory():
+        error 'asStatusHistory() is not supported for log metrics.',
+      withOffset():
+        error 'withOffset() is not supported for log metrics.',
+      withTopK():
+        error 'withTopK() is not supported for log metrics.',
+    },
+
+}

common-lib/common/signal/signal.libsonnet

@@ -5,6 +5,7 @@ local counter = import './counter.libsonnet';
 local gauge = import './gauge.libsonnet';
 local histogram = import './histogram.libsonnet';
 local info = import './info.libsonnet';
+local log = import './log.libsonnet';
 local raw = import './raw.libsonnet';
 local stub = import './stub.libsonnet';
 {
@@ -31,8 +32,8 @@ local stub = import './stub.libsonnet';
   // DEPRECATED. Use unmarshallJsonMulti instead.
   unmarshallJson(signalsJson):
     self.init(
-      datasource=std.get(signalsJson, 'datasource', 'datasource'),
-      datasourceLabel=std.get(signalsJson, 'datasourceLabel', 'Data source'),
+      datasource=std.get(signalsJson, 'datasource', if std.get(signalsJson, 'enableLokiLogs', false) then 'prometheus_datasource' else 'datasource'),
+      datasourceLabel=std.get(signalsJson, 'datasourceLabel', if std.get(signalsJson, 'enableLokiLogs', false) then 'Prometheus data source' else 'Data source'),
       filteringSelector=[signalsJson.filteringSelector],
       groupLabels=signalsJson.groupLabels,
       instanceLabels=signalsJson.instanceLabels,
@@ -46,6 +47,7 @@ local stub = import './stub.libsonnet';
       rangeFunction=std.get(signalsJson, 'rangeFunction', 'rate'),  // rate, irate , delta, increase, idelta...
       varAdHocEnabled=std.get(signalsJson, 'varAdHocEnabled', false),
       varAdHocLabels=std.get(signalsJson, 'varAdHocLabels', []),
+      enableLokiLogs=std.get(signalsJson, 'enableLokiLogs', false),
     )
     +
     {
@@ -85,8 +87,8 @@ local stub = import './stub.libsonnet';
       );
 
     self.init(
-      datasource=std.get(signalsJson, 'datasource', 'datasource'),
-      datasourceLabel=std.get(signalsJson, 'datasourceLabel', 'Data source'),
+      datasource=std.get(signalsJson, 'datasource', if std.get(signalsJson, 'enableLokiLogs', false) then 'prometheus_datasource' else 'datasource'),
+      datasourceLabel=std.get(signalsJson, 'datasourceLabel', if std.get(signalsJson, 'enableLokiLogs', false) then 'Prometheus data source' else 'Data source'),
       filteringSelector=[signalsJson.filteringSelector],
       groupLabels=signalsJson.groupLabels,
       instanceLabels=signalsJson.instanceLabels,
@@ -100,6 +102,7 @@ local stub = import './stub.libsonnet';
       rangeFunction=std.get(signalsJson, 'rangeFunction', std.get(signalsJson, 'rangeFunction', 'rate')),  // rate, irate , delta, increase, idelta...
       varAdHocEnabled=std.get(signalsJson, 'varAdHocEnabled', false),
       varAdHocLabels=std.get(signalsJson, 'varAdHocLabels', []),
+      enableLokiLogs=std.get(signalsJson, 'enableLokiLogs', false),
     )
     +
     {
@@ -177,10 +180,12 @@ local stub = import './stub.libsonnet';
     rangeFunction='rate',
     varAdHocEnabled=false,
     varAdHocLabels=[],
+    enableLokiLogs=false,
   ): self {
 
     local this = self,
-    datasource:: datasource,
+    datasource:: if enableLokiLogs && datasource == 'datasource' then 'prometheus_datasource' else datasource,
+    datasourceLabel:: if enableLokiLogs && datasourceLabel == 'Data source' then 'Prometheus data source' else datasourceLabel,
     aggLevel:: aggLevel,
     aggKeepLabels:: aggKeepLabels,
     aggFunction:: aggFunction,
@@ -191,10 +196,11 @@ local stub = import './stub.libsonnet';
       groupLabels,
       instanceLabels,
       varMetric=varMetric,
-      prometheusDatasourceName=datasource,
-      prometheusDatasourceLabel=datasourceLabel,
+      prometheusDatasourceName=this.datasource,
+      prometheusDatasourceLabel=this.datasourceLabel,
       adHocEnabled=varAdHocEnabled,
       adHocLabels=varAdHocLabels,
+      enableLokiLogs=enableLokiLogs,
     ),
     // vars are used in templating(legend+expressions)
     templatingVariables: {
@@ -216,7 +222,7 @@ local stub = import './stub.libsonnet';
       grafanaVariables.datasources[type],
 
     //name: metric simple name
-    //type: counter, gauge, histogram, // TODO: info metric, status_map metric....
+    //type: counter, gauge, histogram, raw, info, log
     //unit: simple unit
     //description: metric description
     //exprTemplate: expression template
@@ -246,7 +252,7 @@ local stub = import './stub.libsonnet';
       std.prune(
         {
           checks: [
-            if (type != 'gauge' && type != 'histogram' && type != 'counter' && type != 'raw' && type != 'info' && type != 'stub') then error "type must be one of 'gauge','histogram','counter','raw','info' Got %s for %s" % [type, name],
+            if (type != 'gauge' && type != 'histogram' && type != 'counter' && type != 'raw' && type != 'info' && type != 'stub' && type != 'log') then error "type must be one of 'gauge','histogram','counter','raw','info','log'. Got %s for %s" % [type, name],
             if (aggLevel != 'none' && aggLevel != 'instance' && aggLevel != 'group') then error "aggLevel must be one of 'group','instance' or 'none'",
           ],
         }
@@ -260,7 +266,7 @@ local stub = import './stub.libsonnet';
           description=description,
           aggLevel=aggLevel,
           aggFunction=aggFunction,
-          datasource=datasource,
+          datasource=this.datasource,
           vars=this.templatingVariables,
           sourceMaps=sourceMaps,
         )
@@ -273,7 +279,7 @@ local stub = import './stub.libsonnet';
           description=description,
           aggLevel=aggLevel,
           aggFunction=aggFunction,
-          datasource=datasource,
+          datasource=this.datasource,
           vars=this.templatingVariables,
           sourceMaps=sourceMaps,
         )
@@ -286,7 +292,7 @@ local stub = import './stub.libsonnet';
           description=description,
           aggLevel=aggLevel,
           aggFunction=aggFunction,
-          datasource=datasource,
+          datasource=this.datasource,
           vars=this.templatingVariables,
           sourceMaps=sourceMaps,
         )
@@ -299,7 +305,20 @@ local stub = import './stub.libsonnet';
           description=description,
           aggLevel=aggLevel,
           aggFunction=aggFunction,
-          datasource=datasource,
+          datasource=this.datasource,
+          vars=this.templatingVariables,
+          sourceMaps=sourceMaps,
+        )
+      else if type == 'log' then
+        log.new(
+          name=name,
+          type=type,
+          unit='none',
+          nameShort=nameShort,
+          description=description,
+          aggLevel=aggLevel,
+          aggFunction=aggFunction,
+          datasource='loki_datasource',
           vars=this.templatingVariables,
           sourceMaps=sourceMaps,
         )
@@ -311,7 +330,7 @@ local stub = import './stub.libsonnet';
           description=description,
           aggLevel=aggLevel,
           aggFunction=aggFunction,
-          datasource=datasource,
+          datasource=this.datasource,
           vars=this.templatingVariables,
           sourceMaps=sourceMaps,
         )

common-lib/common/signal/test_counter_with_enableLokiLogs.libsonnet

@@ -0,0 +1,73 @@
+local signal = import './signal.libsonnet';
+local test = import 'jsonnetunit/test.libsonnet';
+
+local m1 = signal.init(
+  filteringSelector=['job="abc"'],
+  interval='5m',
+  aggLevel='instance',
+  aggFunction='max',
+  enableLokiLogs=true,
+).addSignal(
+  name='API server requests',
+  nameShort='requests',
+  type='counter',
+  unit='requests',
+  description='API server calls.',
+  sourceMaps=[
+    {
+      expr: 'apiserver_request_total{%(queriesSelector)s}',
+      rangeFunction: 'rate',
+      aggKeepLabels: [],
+      legendCustomTemplate: null,
+      infoLabel: null,
+    },
+  ],
+)
+;
+
+{
+
+  asTarget: {
+    local raw = m1.asTarget(),
+    testResult: test.suite({
+      testLegend: {
+        actual: raw.legendFormat,
+        expect: '{{instance}}: requests',
+      },
+      testExpression: {
+        actual: raw.expr,
+        expect: 'max by (job,instance) (\n  rate(apiserver_request_total{job="abc",job=~"$job",instance=~"$instance"}[5m])\n)',
+      },
+    }),
+  },
+  asTimeSeries:
+    {
+      local raw = m1.asTimeSeries(),
+      testResult: test.suite({
+        testTStitle: {
+          actual: raw.title,
+          expect: 'API server requests',
+        },
+        testUnit: {
+          actual: raw.fieldConfig.overrides[0].properties[0].value,
+          expect: 'rps',
+        },
+        testTStype: {
+          actual: raw.type,
+          expect: 'timeseries',
+        },
+        testTSversion: {
+          actual: raw.pluginVersion,
+          expect: 'v11.0.0',
+        },
+        testTSUid: {
+          actual: raw.datasource,
+          expect: {
+            uid: '${prometheus_datasource}',
+            type: 'prometheus',
+          },
+        },
+      }),
+    },
+
+}

common-lib/common/signal/test_log.libsonnet

@@ -0,0 +1,44 @@
+local signal = import './signal.libsonnet';
+local test = import 'jsonnetunit/test.libsonnet';
+
+local m1 = signal.init(
+  filteringSelector=['foo="bar"'],
+  interval='10m',
+  aggLevel='group',
+  aggFunction='avg',
+).addSignal(
+  name='Log query',
+  nameShort='Log',
+  type='log',
+  unit='none',
+  description='Log query.',
+  sourceMaps=[
+    {
+      expr: '{%(queriesSelector)s} | json',
+      rangeFunction: 'rate',
+      aggKeepLabels: [],
+      legendCustomTemplate: null,
+      infoLabel: null,
+    },
+  ],
+
+);
+
+{
+  asTarget: {
+    local raw = m1.asTarget(),
+    testResult: test.suite({
+      testExpression: {
+        actual: raw.expr,
+        expect: '{foo="bar",job=~"$job",instance=~"$instance"} | json',
+      },
+      testTSUid: {
+        actual: raw.datasource,
+        expect: {
+          uid: '${loki_datasource}',
+          type: 'loki',
+        },
+      },
+    }),
+  },
+}