Merge branch 'main' into configure-head-chunks-write-queue-size

Author: friedrichg

.github/workflows/ci.yaml

@@ -9,7 +9,7 @@ on:
 jobs:
   lint:
     runs-on: ubuntu-latest
-    container: quay.io/cortexproject/cortex-jsonnet-build-image:e63d87f
+    container: quay.io/cortexproject/cortex-jsonnet-build-image:e158eda
     steps:
       - uses: actions/checkout@v2
         name: Checkout
@@ -23,7 +23,7 @@ jobs:
         run: make lint-playbooks
   build:
     runs-on: ubuntu-latest
-    container: quay.io/cortexproject/cortex-jsonnet-build-image:e63d87f
+    container: quay.io/cortexproject/cortex-jsonnet-build-image:e158eda
     steps:
       - uses: actions/checkout@v2
         name: Checkout
@@ -34,7 +34,7 @@ jobs:
         run: make build-mixin
   readme:
     runs-on: ubuntu-latest
-    container: quay.io/cortexproject/cortex-jsonnet-build-image:e63d87f
+    container: quay.io/cortexproject/cortex-jsonnet-build-image:e158eda
     steps:
       - uses: actions/checkout@v2
         name: Checkout

CHANGELOG.md

@@ -1,6 +1,18 @@
 # Changelog
 
 ## master / unreleased
+* [CHANGE] Upgrade memcached to 1.6.23-alpine and memcached-exporter to v0.14.2
+* [CHANGE] Use cortex v1.16.0
+* [ENHANCEMENT] Enable frontend query stats by default
+* [ENHANCEMENT] Enable ruler query stats by default
+
+## 1.15.3 / 2023-11-24
+* [CHANGE] Add default instance max series for ingesters
+* [CHANGE] Add default instance max inflight pushes for distributors
+* [CHANGE] Remove mem-ballast from distributor and querier.
+* [CHANGE] Increase cpu requests for querier to 2.
+* [CHANGE] Configure GOMAXPROCS and GOMEMLIMIT for all cortex modules based on cpu and memory requests or limits
+* [CHANGE] Add default tenant shard sizes
 * [CHANGE] Use cortex v1.15.3
 * [CHANGE] Azure storage endpoint suffix is set to `blob.core.windows.net` for backward compatibility
 * [ENHANCEMENT] Configure head_chunks_write_queue_size to 1 million to avoid high latency when shipping

README.md

@@ -17,15 +17,15 @@ To generate the YAMLs for deploying Cortex:
 
     ```console
     $ # make sure to be outside of GOPATH or a go.mod project
-    $ GO111MODULE=on go install github.com/grafana/tanka/cmd/tk@v0.24.0
+    $ GO111MODULE=on go install github.com/grafana/tanka/cmd/tk@v0.26.0
     $ GO111MODULE=on go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/[email protected]
     ```
 
 1. Initialise the Tanka repo, install the Cortex and Kubernetes Jsonnet libraries.
 
     ```console
     $ mkdir <name> && cd <name>
-    $ tk init --k8s=1.24 # this includes github.com/jsonnet-libs/k8s-libsonnet/1.24@main
+    $ tk init --k8s=1.26 # this includes github.com/jsonnet-libs/k8s-libsonnet/1.26@main
     $ jb install github.com/cortexproject/cortex-jsonnet/cortex@main
     ```
 

build-image/Dockerfile

@@ -1,5 +1,5 @@
 # Build jsonnet
-FROM alpine:3.17 AS jsonnet-builder
+FROM alpine:3.18 AS jsonnet-builder
 RUN apk add --no-cache git make g++
 RUN git clone https://github.com/google/jsonnet && \
     git  -C jsonnet checkout v0.20.0 && \
@@ -8,7 +8,7 @@ RUN git clone https://github.com/google/jsonnet && \
     cp jsonnet/jsonnetfmt /usr/bin
 
 # Build jb
-FROM alpine:3.17 AS jb-builder
+FROM alpine:3.18 AS jb-builder
 ARG JSONNET_BUNDLER_VERSION=0.5.1
 ARG JSONNET_BUNDLER_CHECKSUM="f5bccc94d28fbbe8ad1d46fd4f208619e45d368a5d7924f6335f4ecfa0605c85  /usr/bin/jb"
 RUN apk add --no-cache curl
@@ -17,19 +17,19 @@ RUN echo "${JSONNET_BUNDLER_CHECKSUM}" | sha256sum -c || (printf "wanted: %s\n
 RUN chmod +x /usr/bin/jb
 
 # Build tanka
-FROM alpine:3.17 AS tk-builder
-ARG TANKA_VERSION=0.24.0
-ARG TANKA_CHECKSUM="82c8c533c29eefea0af9c28f487203b19dec84ce2624702f99196e777f946ddc  /usr/bin/tk"
+FROM alpine:3.18 AS tk-builder
+ARG TANKA_VERSION=0.26.0
+ARG TANKA_CHECKSUM="089796ae2ce65390501b2c68ceca1ce99ff12787d5ae3b4823c825a07e6e22f4  /usr/bin/tk"
 RUN apk add --no-cache curl
 RUN curl -fSL -o "/usr/bin/tk" "https://github.com/grafana/tanka/releases/download/v${TANKA_VERSION}/tk-linux-amd64"
 RUN echo "${TANKA_CHECKSUM}" | sha256sum -c || (printf "wanted: %s\n   got: %s\n" "${TANKA_CHECKSUM}" "$(sha256sum /usr/bin/tk)"; exit 1)
 RUN chmod +x /usr/bin/tk
 
 # Build mixtool
-FROM golang:1.20-alpine AS mixtool-builder
+FROM golang:1.21-alpine AS mixtool-builder
 RUN GO111MODULE=on go install github.com/monitoring-mixins/mixtool/cmd/mixtool@ae18e31161ea10545b9c1ac0d23c10122f2c12b5
 
-FROM alpine:3.17
+FROM alpine:3.18
 RUN apk add --no-cache git make libgcc libstdc++ zip findutils sed yq
 COPY --from=jsonnet-builder /usr/bin/jsonnetfmt /usr/bin
 COPY --from=jsonnet-builder /usr/bin/jsonnet /usr/bin

cortex/alertmanager.libsonnet

@@ -3,6 +3,7 @@
   local volumeMount = $.core.v1.volumeMount,
   local volume = $.core.v1.volume,
   local container = $.core.v1.container,
+  local envType = container.envType,
   local statefulSet = $.apps.v1.statefulSet,
   local service = $.core.v1.service,
   local configMap = $.core.v1.configMap,
@@ -96,6 +97,7 @@
     if $._config.alertmanager_enabled then
       container.new('alertmanager', $._images.alertmanager) +
       container.withPorts($.util.defaultPorts + mode.ports) +
+      container.withEnvMap($.alertmanager_env_map) +
       container.withEnvMixin([container.envType.fromFieldPath('POD_IP', 'status.podIP')]) +
       container.withArgsMixin(
         $.util.mapToFlags($.alertmanager_args) +
@@ -109,9 +111,13 @@
       ) +
       $.util.resourcesRequests('100m', '1Gi') +
       $.util.readinessProbe +
+      $.go_container_mixin +
       $.jaeger_mixin
     else {},
 
+  alertmanager_env_map:: {
+  },
+
   alertmanager_statefulset:
     if $._config.alertmanager_enabled then
       statefulSet.new('alertmanager', $._config.alertmanager.replicas, [$.alertmanager_container], $.alertmanager_pvc) +

cortex/compactor.libsonnet

@@ -1,5 +1,6 @@
 {
   local container = $.core.v1.container,
+  local envType = container.envType,
   local pvc = $.core.v1.persistentVolumeClaim,
   local statefulSet = $.apps.v1.statefulSet,
   local volumeMount = $.core.v1.volumeMount,
@@ -43,13 +44,18 @@
     container.new('compactor', $._images.compactor) +
     container.withPorts($.compactor_ports) +
     container.withArgsMixin($.util.mapToFlags($.compactor_args)) +
+    container.withEnvMap($.compactor_env_map) +
     container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) +
     // Do not limit compactor CPU and request enough cores to honor configured max concurrency.
-    $.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '6Gi') +
+    $.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '5Gi') +
     $.util.resourcesLimits(null, '6Gi') +
     $.util.readinessProbe +
+    $.go_container_mixin +
     $.jaeger_mixin,
 
+  compactor_env_map:: {
+  },
+
   newCompactorStatefulSet(name, container)::
     statefulSet.new(name, 1, [container], compactor_data_pvc) +
     statefulSet.mixin.spec.withServiceName(name) +

cortex/config.libsonnet

@@ -260,6 +260,8 @@
 
         // No retention for now.
         compactor_blocks_retention_period: '0',
+
+        ingestion_tenant_shard_size: 3,
       },
 
       medium_small_user:: {
@@ -277,6 +279,8 @@
         // 1000 rules
         ruler_max_rules_per_rule_group: 20,
         ruler_max_rule_groups_per_tenant: 50,
+
+        ingestion_tenant_shard_size: 9,
       },
 
       small_user:: {
@@ -294,6 +298,8 @@
         // 1400 rules
         ruler_max_rules_per_rule_group: 20,
         ruler_max_rule_groups_per_tenant: 70,
+
+        ingestion_tenant_shard_size: 15,
       },
 
       medium_user:: {
@@ -311,6 +317,8 @@
         // 1800 rules
         ruler_max_rules_per_rule_group: 20,
         ruler_max_rule_groups_per_tenant: 90,
+
+        ingestion_tenant_shard_size: 30,
       },
 
       big_user:: {
@@ -328,6 +336,8 @@
         // 2200 rules
         ruler_max_rules_per_rule_group: 20,
         ruler_max_rule_groups_per_tenant: 110,
+
+        ingestion_tenant_shard_size: 60,
       },
 
       super_user:: {
@@ -345,6 +355,8 @@
         // 2600 rules
         ruler_max_rules_per_rule_group: 20,
         ruler_max_rule_groups_per_tenant: 130,
+
+        ingestion_tenant_shard_size: 120,
       },
 
       // This user class has limits increased by +50% compared to the previous one.
@@ -363,6 +375,8 @@
         // 3000 rules
         ruler_max_rules_per_rule_group: 20,
         ruler_max_rule_groups_per_tenant: 150,
+
+        ingestion_tenant_shard_size: 180,
       },
     },
 
@@ -381,15 +395,41 @@
     ingester_stream_chunks_when_using_blocks: true,
 
     // Ingester limits are put directly into runtime config, if not null. Available limits:
-    //    ingester_instance_limits: {
-    //      max_inflight_push_requests: 0,  // Max inflight push requests per ingester. 0 = no limit.
-    //      max_ingestion_rate: 0,  // Max ingestion rate (samples/second) per ingester. 0 = no limit.
-    //      max_series: 0,  // Max number of series per ingester. 0 = no limit.
-    //      max_tenants: 0,  // Max number of tenants per ingester. 0 = no limit.
-    //    },
-    ingester_instance_limits: null,
+    ingester_instance_limits: {
+      // max_inflight_push_requests: 0,  // Max inflight push requests per ingester. 0 = no limit.
+      // max_ingestion_rate: 0,  // Max ingestion rate (samples/second) per ingester. 0 = no limit.
+      max_series: 4.8e+6,  // Max number of series per ingester. 0 = no limit. 4.8 million is closely tied to 15Gb in requests per ingester
+      // max_tenants: 0,  // Max number of tenants per ingester. 0 = no limit.
+    },
+
+    // if we disable this, we need to make sure we set the resource limits
+    // Disabling this can potentially increase cortex performance,
+    // but it will also cause performance inconsistencies
+    gomaxprocs_based_on_cpu_requests: true,
+    gomemlimit_based_on_mem_requests: true,
+
+    gomaxprocs_resource:
+      if $._config.gomaxprocs_based_on_cpu_requests then
+        'requests.cpu'
+      else
+        'limits.cpu',
+
+    gomemlimit_resource:
+      if $._config.gomemlimit_based_on_mem_requests then
+        'requests.memory'
+      else
+        'limits.memory',
   },
 
+  go_container_mixin::
+    local container = $.core.v1.container;
+    container.withEnvMixin([
+      container.envType.withName('GOMAXPROCS') +
+      container.envType.valueFrom.resourceFieldRef.withResource($._config.gomaxprocs_resource),
+      container.envType.withName('GOMEMLIMIT') +
+      container.envType.valueFrom.resourceFieldRef.withResource($._config.gomemlimit_resource),
+    ]),
+
   local configMap = $.core.v1.configMap,
 
   overrides_config:

cortex/distributor.libsonnet

@@ -1,5 +1,6 @@
 {
   local container = $.core.v1.container,
+  local envType = container.envType,
   local containerPort = $.core.v1.containerPort,
 
   distributor_args::
@@ -20,11 +21,6 @@
       'distributor.ha-tracker.etcd.endpoints': 'etcd-client.%s.svc.cluster.local.:2379' % $._config.namespace,
       'distributor.ha-tracker.prefix': 'prom_ha/',
 
-      // The memory requests are 2G, and we barely use 100M.
-      // By adding a ballast of 1G, we can drastically reduce GC, but also keep the usage at
-      // around 1.25G, reducing the 99%ile.
-      'mem-ballast-size-bytes': 1 << 30,  // 1GB
-
       'server.grpc.keepalive.max-connection-age': '2m',
       'server.grpc.keepalive.max-connection-age-grace': '5m',
       'server.grpc.keepalive.max-connection-idle': '1m',
@@ -36,17 +32,23 @@
       // Do not extend the replication set on unhealthy (or LEAVING) ingester when "unregister on shutdown"
       // is set to false.
       'distributor.extend-writes': $._config.unregister_ingesters_on_shutdown,
+      'distributor.instance-limits.max-inflight-push-requests': 60,  //60 is very conservative to protect the distributor from OOMs
     },
 
+  distributor_env_map:: {
+  },
+
   distributor_ports:: $.util.defaultPorts,
 
   distributor_container::
     container.new('distributor', $._images.distributor) +
     container.withPorts($.distributor_ports) +
     container.withArgsMixin($.util.mapToFlags($.distributor_args)) +
+    container.withEnvMap($.distributor_env_map) +
     $.util.resourcesRequests('2', '2Gi') +
     $.util.resourcesLimits(null, '4Gi') +
     $.util.readinessProbe +
+    $.go_container_mixin +
     $.jaeger_mixin,
 
   local deployment = $.apps.v1.deployment,

cortex/flusher-job-blocks.libsonnet

@@ -21,11 +21,16 @@
       target: 'flusher',
       'blocks-storage.tsdb.retention-period': '10000h',  // don't delete old blocks too soon.
     })) +
+    container.withEnvMap($.flusher_env_map) +
     $.util.resourcesRequests('4', '15Gi') +
     $.util.resourcesLimits(null, '25Gi') +
     $.util.readinessProbe +
+    $.go_container_mixin +
     $.jaeger_mixin,
 
+  flusher_env_map:: {
+  },
+
   flusher_job_func(jobName, pvcName)::
     job.new() +
     job.mixin.spec.template.spec.withContainers([

cortex/images.libsonnet

@@ -1,11 +1,11 @@
 {
   _images+:: {
     // Various third-party images.
-    memcached: 'memcached:1.6.9-alpine',
-    memcachedExporter: 'prom/memcached-exporter:v0.6.0',
+    memcached: 'memcached:1.6.23-alpine',
+    memcachedExporter: 'prom/memcached-exporter:v0.14.2',
 
     // Our services.
-    cortex: 'cortexproject/cortex:v1.15.3',
+    cortex: 'cortexproject/cortex:v1.16.0',
 
     alertmanager: self.cortex,
     distributor: self.cortex,
@@ -20,7 +20,7 @@
     query_scheduler: self.cortex,
 
     overrides_exporter: self.cortex,
-    query_tee: 'quay.io/cortexproject/query-tee:v1.15.3',
-    testExporter: 'cortexproject/test-exporter:v1.15.3',
+    query_tee: 'quay.io/cortexproject/query-tee:v1.16.0',
+    testExporter: 'cortexproject/test-exporter:v1.16.0',
   },
 }