Merge branch 'master' into flusher-job-blocks

Author: pstibrany

.circleci/config.yml

@@ -11,22 +11,22 @@ workflows:
 jobs:
   lint:
     docker:
-      - image: grafana/cortex-jsonnet-build-image:55f5699
+      - image: grafana/cortex-jsonnet-build-image:8ce0de1
     steps:
       - checkout
       - run: make lint
 
   build:
     docker:
-      - image: grafana/cortex-jsonnet-build-image:55f5699
+      - image: grafana/cortex-jsonnet-build-image:8ce0de1
     steps:
       - checkout
       - run: make build-mixin
       - store_artifacts:
-          path: cortex-mixin.zip
+          path: cortex-mixin/cortex-mixin.zip
   test-readme:
     docker:
-      - image: grafana/cortex-jsonnet-build-image:55f5699
+      - image: grafana/cortex-jsonnet-build-image:8ce0de1
     steps:
       - checkout
       - run: make test-readme

CHANGELOG.md

@@ -2,18 +2,30 @@
 
 ## master / unreleased
 
+* [CHANGE] Add the default preset 'extra_small_user' and reference it in the CLI flags. This will raise the limits of the 'small_user' preset to the defaults for `ingester.max-samples-per-query` and `ingester.max-series-per-query`. #200
+* [ENHANCEMENT] Add the Ruler to the read resources dashboard #205
+* [ENHANCEMENT] Read dashboards now use `cortex_querier_request_duration_seconds` metrics to allow for accurate dashboards when deploying Cortex as a single-binary. #199
+* [ENHANCEMENT] Improved Ruler dashboard. Includes information about notifications, reads/writes, and per user per rule group evaluation. #197, #205
+* [ENHANCEMENT] Add `flusher-job-blocks.libsonnet` with support for flushing blocks disks. #187
+* [FEATURE] Latency recording rules for the metric`cortex_querier_request_duration_seconds` are now part of a `cortex_querier_api` rule group. #199
+* [FEATURE] Add overrides-exporter as optional deployment to expose configured runtime overrides and presets. #198
+* [FEATURE] Add a dashboard for the alertmanager. #207
+* [BUGFIX] Added `ingester-blocks` to ingester's job label matcher, in order to correctly get metrics when migrating a Cortex cluster from chunks to blocks. #203
+
+## 1.4.0 / 2020-10-02
+
 * [CHANGE] Lower the default overrides configs for ingestion and create a new overrides user out of the previous config #183
 * [CHANGE] The project is now licensed with Apache-2.0 license. #169
 * [CHANGE] Add overrides config to tsdb store-gateway. #167
 * [CHANGE] Ingesters now default to running as `StatefulSet` with WAL enabled. It is controlled by the config `$._config.ingester_deployment_without_wal` which is `false` by default. Setting the config to `true` will yield the old behaviour (stateless `Deployment` without WAL enabled). #72
 * [CHANGE] We now allow queries that are 32 days long. For example, rate(metric[32d]). Before it was 31d. #173
 * [CHANGE] Renamed `container_name` and `pod_name` label names to `container` and `pod` respectively. This is required in order to comply with cAdvisor metrics changes shipped with Kubernetes 1.16. #179
 * [CHANGE] Removed the `experimental` prefix from blocks storage CLI flags. #179
+* [CHANGE] Rename flags `store-gateway.replication-factor` and `store-gateway.tokens-file-path` to `store-gateway.sharding-ring.replication-factor` and `store-gateway.sharding-ring.tokens-file-path` in anticipation of v1.4 release. #191
 * [ENHANCEMENT] Enable support for HA in the Cortex Alertmanager #147
 * [ENHANCEMENT] Support `alertmanager.fallback_config` option in the Alertmanager. #179
 * [ENHANCEMENT] Add support for S3 block storage. #181
-* [ENHANCEMENT] Add support for Azure block storage. #182
-* [ENHANCEMENT] Add `flusher-job-blocks.libsonnet` with support for flushing blocks disks. #187
+* [ENHANCEMENT] Add support for Azure block storage. #182 #190
 * [BUGFIX] Add support the `local` ruler client type  #175
 * [BUGFIX] Fixes `ruler.storage.s3.url` argument for the Ruler. It used an incorrect argument. #177
 

Makefile

@@ -2,13 +2,18 @@
 
 JSONNET_FMT := jsonnetfmt
 
-lint:
+lint: lint-mixin
 	@RESULT=0; \
 	for f in $$(find . -name '*.libsonnet' -print -o -name '*.jsonnet' -print); do \
 		$(JSONNET_FMT) -- "$$f" | diff -u "$$f" -; \
 		RESULT=$$(($$RESULT + $$?)); \
 	done; \
-	exit $$RESULT
+	RESULT=$$(($$RESULT + $$?)); \
+
+lint-mixin:
+	cd cortex-mixin && \
+	jb install && \
+	mixtool lint mixin.libsonnet
 
 fmt:
 	@find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
@@ -24,10 +29,8 @@ build-mixin:
 	cd cortex-mixin && \
 	rm -rf out && mkdir out && \
 	jb install && \
-	jsonnet -J vendor -S dashboards.jsonnet -m out/ && \
-	jsonnet -J vendor -S recording_rules.jsonnet > out/rules.yaml && \
-	jsonnet -J vendor -S alerts.jsonnet > out/alerts.yaml
-	zip -r cortex-mixin.zip cortex-mixin/out
+	mixtool generate all --output-alerts out/alerts.yaml --output-rules out/rules.yaml --directory out/dashboards mixin.libsonnet && \
+	zip -r cortex-mixin.zip out
 
 test-readme:
 	rm -rf test-readme && \

build-image/Dockerfile

@@ -25,9 +25,14 @@ RUN curl -fSL -o "/usr/bin/tk" "https://github.com/grafana/tanka/releases/downlo
 RUN echo "${TANKA_CHECKSUM}" | sha256sum -c || (printf "wanted: %s\n   got: %s\n" "${TANKA_CHECKSUM}" "$(sha256sum /usr/bin/tk)"; exit 1)
 RUN chmod +x /usr/bin/tk
 
+# Build mixtool
+FROM golang:1.15-alpine AS mixtool-builder
+RUN GO111MODULE=on go get github.com/monitoring-mixins/mixtool/cmd/mixtool@59d44357240d
+
 FROM alpine:3.11
 RUN apk add --no-cache git make libgcc libstdc++ zip
 COPY --from=jsonnet-builder /usr/bin/jsonnetfmt /usr/bin
 COPY --from=jsonnet-builder /usr/bin/jsonnet /usr/bin
 COPY --from=jb-builder /usr/bin/jb /usr/bin
 COPY --from=tk-builder /usr/bin/tk /usr/bin
+COPY --from=mixtool-builder /go/bin/mixtool /usr/bin

cortex-mixin/alerts.jsonnet

@@ -26,9 +26,10 @@
     // These are used by the dashboards and allow for the simultaneous display of
     // microservice and single binary cortex clusters.
     job_names: {
-      ingester: '(ingester|cortex$)',
+      ingester: '(ingester.*|cortex$)',  // Match also ingester-blocks, which is used during the migration from chunks to blocks.
       distributor: '(distributor|cortex$)',
       querier: '(querier|cortex$)',
+      ruler: '(ruler|cortex$)',
       query_frontend: '(query-frontend|cortex$)',
       table_manager: '(table-manager|cortex$)',
       store_gateway: '(store-gateway|cortex$)',

cortex-mixin/config.libsonnet

@@ -4,6 +4,7 @@
     (import 'dashboards/queries.libsonnet') +
     (import 'dashboards/reads.libsonnet') +
     (import 'dashboards/ruler.libsonnet') +
+    (import 'dashboards/alertmanager.libsonnet') +
     (import 'dashboards/scaling.libsonnet') +
     (import 'dashboards/writes.libsonnet') +
 

cortex-mixin/dashboards.jsonnet

@@ -0,0 +1,88 @@
+local utils = import 'mixin-utils/utils.libsonnet';
+
+(import 'dashboard-utils.libsonnet') {
+
+  'alertmanager.json':
+    ($.dashboard('Cortex / Alertmanager') + { uid: 'a76bee5913c97c918d9e56a3cc88cc28' })
+    .addClusterSelectorTemplates()
+    .addRow(
+      ($.row('Headlines') + {
+         height: '100px',
+         showTitle: false,
+       })
+      .addPanel(
+        $.panel('Total Alerts') +
+        $.statPanel('sum(cortex_alertmanager_alerts{%s})' % $.jobMatcher('alertmanager'), format='short')
+      )
+      .addPanel(
+        $.panel('Total Silences') +
+        $.statPanel('sum(cortex_alertmanager_silences{%s})' % $.jobMatcher('alertmanager'), format='short')
+      )
+    )
+    .addRow(
+      $.row('Alerts Received')
+      .addPanel(
+        $.panel('APS') +
+        $.queryPanel(
+          [
+            |||
+              sum(rate(cortex_alertmanager_alerts_received_total{%s}[$__interval]))
+              -
+              sum(rate(cortex_alertmanager_alerts_invalid_total{%s}[$__interval]))
+            ||| % [$.jobMatcher('alertmanager'), $.jobMatcher('alertmanager')],
+            'sum(rate(cortex_alertmanager_alerts_invalid_total{%s}[$__interval]))' % $.jobMatcher('alertmanager'),
+          ],
+          ['success', 'failed']
+        )
+      )
+    )
+    .addRow(
+      $.row('Alert Notifications')
+      .addPanel(
+        $.panel('NPS') +
+        $.queryPanel(
+          [
+            |||
+              sum(rate(cortex_alertmanager_notifications_total{%s}[$__interval]))
+              -
+              sum(rate(cortex_alertmanager_notifications_failed_total{%s}[$__interval]))
+            ||| % [$.jobMatcher('alertmanager'), $.jobMatcher('alertmanager')],
+            'sum(rate(cortex_alertmanager_notifications_failed_total{%s}[$__interval]))' % $.jobMatcher('alertmanager'),
+          ],
+          ['success', 'failed']
+        )
+      )
+      .addPanel(
+        $.panel('NPS by integration') +
+        $.queryPanel(
+          [
+            |||
+              (
+              sum(rate(cortex_alertmanager_notifications_total{%s}[$__interval])) by(integration)
+              -
+              sum(rate(cortex_alertmanager_notifications_failed_total{%s}[$__interval])) by(integration)
+              ) > 0
+              or on () vector(0)
+            ||| % [$.jobMatcher('alertmanager'), $.jobMatcher('alertmanager')],
+            'sum(rate(cortex_alertmanager_notifications_failed_total{%s}[$__interval])) by(integration)' % $.jobMatcher('alertmanager'),
+          ],
+          ['success - {{ integration }}', 'failed - {{ integration }}']
+        )
+      )
+      .addPanel(
+        $.panel('Latency') +
+        $.latencyPanel('cortex_alertmanager_notification_latency_seconds', '{%s}' % $.jobMatcher('alertmanager'))
+      )
+    )
+    .addRow(
+      $.row('Configuration API (gateway) + Alertmanager UI')
+      .addPanel(
+        $.panel('QPS') +
+        $.qpsPanel('cortex_request_duration_seconds_count{%s, route=~"api_v1_alerts|alertmanager"}' % $.jobMatcher($._config.job_names.gateway))
+      )
+      .addPanel(
+        $.panel('Latency') +
+        utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.gateway) + [utils.selector.re('route', 'api_v1_alerts|alertmanager')])
+      )
+    ),
+}

cortex-mixin/dashboards.libsonnet

@@ -2,7 +2,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
 
 (import 'dashboard-utils.libsonnet') {
   'cortex-chunks.json':
-    $.dashboard('Cortex / Chunks')
+    ($.dashboard('Cortex / Chunks') + { uid: 'a56a3fa6284064eb392a115f3acbf744' })
     .addClusterSelectorTemplates()
     .addRow(
       $.row('Active Series / Chunks')
@@ -52,7 +52,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
     ),
 
   'cortex-wal.json':
-    $.dashboard('Cortex / WAL')
+    ($.dashboard('Cortex / WAL') + { uid: 'd4fb924cdc1581cd8e870e3eb0110bda' })
     .addClusterSelectorTemplates()
     .addRow(
       $.row('')