Merge pull request grafana#394 from stevesg/josh-alertmanager

Alertmanager: Add sharding configuration.

Author: stevesg

CHANGELOG.md

@@ -55,6 +55,7 @@
   * `CortexAlertmanagerPersistStateFailing`
   * `CortexAlertmanagerInitialSyncFailed`
 * [ENHANCEMENT] Add support for Azure storage in Alertmanager configuration. #381
+* [ENHANCEMENT] Add support for running Alertmanager in sharding mode. #394
 * [BUGFIX] Fixed `CortexIngesterHasNotShippedBlocks` alert false positive in case an ingester instance had ingested samples in the past, then no traffic was received for a long period and then it started receiving samples again. #308
 * [BUGFIX] Alertmanager: fixed `--alertmanager.cluster.peers` CLI flag passed to alertmanager when HA is enabled. #329
 * [BUGFIX] Fixed `CortexInconsistentRuntimeConfig` metric. #335

cortex/alertmanager.libsonnet

@@ -7,18 +7,62 @@
   local service = $.core.v1.service,
   local configMap = $.core.v1.configMap,
 
-  local isHA = $._config.alertmanager.replicas > 1,
+  // The Alertmanager has three operational modes.
+  local haType = if $._config.alertmanager.sharding_enabled then
+    'sharding'
+  else if $._config.alertmanager.replicas > 1 then
+    'gossip_multi_replica'
+  else
+    'gossip_single_replica',
+  // mode represents which operational mode the alertmanager runs in.
+  // ports: array of container ports used for gossiping.
+  // args: arguments that are eventually converted to flags on the container
+  // flags: arguments directly added to the container. For legacy reasons, we need to use -- as a prefix for some flags.
+  // service: the service definition
+  local mode = {
+    sharding: {
+      ports: [],
+      args: {
+        'alertmanager.sharding-enabled': true,
+        'alertmanager.sharding-ring.store': $._config.alertmanager.ring_store,
+        'alertmanager.sharding-ring.consul.hostname': $._config.alertmanager.ring_hostname,
+        'alertmanager.sharding-ring.replication-factor': $._config.alertmanager.ring_replication_factor,
+      },
+      flags: [],
+      service:
+        $.util.serviceFor($.alertmanager_statefulset) +
+        service.mixin.spec.withClusterIp('None'),
+    },
+    gossip_multi_replica: {
+      ports: [
+        $.core.v1.containerPort.newUDP('gossip-udp', $._config.alertmanager.gossip_port),
+        $.core.v1.containerPort.new('gossip-tcp', $._config.alertmanager.gossip_port),
+      ],
+      args: {},
+      flags: [
+        '--alertmanager.cluster.listen-address=[$(POD_IP)]:%s' % $._config.alertmanager.gossip_port,
+        '--alertmanager.cluster.peers=%s' % std.join(',', peers),
+      ],
+      service:
+        $.util.serviceFor($.alertmanager_statefulset) +
+        service.mixin.spec.withClusterIp('None'),
+    },
+    gossip_single_replica: {
+      ports: [],
+      args: {},
+      flags: ['--alertmanager.cluster.listen-address=""'],
+      service: $.util.serviceFor($.alertmanager_statefulset),
+    },
+  }[haType],
   local hasFallbackConfig = std.length($._config.alertmanager.fallback_config) > 0,
-  local peers = if isHA then
-    [
-      'alertmanager-%d.alertmanager.%s.svc.%s.local:%s' % [i, $._config.namespace, $._config.cluster, $._config.alertmanager.gossip_port]
-      for i in std.range(0, $._config.alertmanager.replicas - 1)
-    ]
-  else [],
-
+  local peers = [
+    'alertmanager-%d.alertmanager.%s.svc.%s.local:%s' % [i, $._config.namespace, $._config.cluster, $._config.alertmanager.gossip_port]
+    for i in std.range(0, $._config.alertmanager.replicas - 1)
+  ],
   alertmanager_args::
     $._config.grpcConfig +
     $._config.alertmanagerStorageClientConfig +
+    mode.args +
     {
       target: 'alertmanager',
       'log.level': 'debug',
@@ -51,24 +95,11 @@
   alertmanager_container::
     if $._config.alertmanager_enabled then
       container.new('alertmanager', $._images.alertmanager) +
-      container.withPorts(
-        $.util.defaultPorts +
-        if isHA then [
-          $.core.v1.containerPort.newUDP('gossip-udp', $._config.alertmanager.gossip_port),
-          $.core.v1.containerPort.new('gossip-tcp', $._config.alertmanager.gossip_port),
-        ]
-        else [],
-      ) +
+      container.withPorts($.util.defaultPorts + mode.ports) +
       container.withEnvMixin([container.envType.fromFieldPath('POD_IP', 'status.podIP')]) +
       container.withArgsMixin(
         $.util.mapToFlags($.alertmanager_args) +
-        (
-          if isHA then
-            ['--alertmanager.cluster.listen-address=[$(POD_IP)]:%s' % $._config.alertmanager.gossip_port] +
-            ['--alertmanager.cluster.peers=%s' % std.join(',', peers)]
-          else
-            ['-alertmanager.cluster.listen-address=""']
-        )
+        mode.flags
       ) +
       container.withVolumeMountsMixin(
         [volumeMount.new('alertmanager-data', '/data')] +
@@ -101,11 +132,5 @@
     else {},
 
   alertmanager_service:
-    if $._config.alertmanager_enabled then
-      if isHA then
-        $.util.serviceFor($.alertmanager_statefulset) +
-        service.mixin.spec.withClusterIp('None')
-      else
-        $.util.serviceFor($.alertmanager_statefulset)
-    else {},
+    if $._config.alertmanager_enabled then mode.service else {},
 }

cortex/config.libsonnet

@@ -276,8 +276,12 @@
 
     alertmanager: {
       replicas: 3,
+      sharding_enabled: false,
       gossip_port: 9094,
       fallback_config: {},
+      ring_store: 'consul',
+      ring_hostname: 'consul.%s.svc.cluster.local:8500' % $._config.namespace,
+      ring_replication_factor: $._config.replication_factor,
     },
 
     alertmanager_client_type: error 'you must specify a storage backend type for the alertmanager (azure, gcs, s3, local)',