Memcached configuration belongs in tsdb-config.libsonnet (#12)

This effectively fixes max idle connections for memcached-metadata in ruler and querier

Author: friedrichg

CHANGELOG.md

@@ -10,6 +10,7 @@
 * [ENHANCEMENT] How to rename buckets in AWS and Azure for `not healthy index found` playbook. #5
 * [ENHANCEMENT] Support new metrics cortex_cache_fetched_keys_total and cortex_cache_fetched_keys_total
 * [BUGFIX] Updated blocks_storage_s3_endpoint in config.libsonnet to include the correct aws region
+* [BUGFIX] Fixes `-blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections` for ruler and querier
 
 ## 1.11.0 / 2021-12-30
 

cortex/store-gateway.libsonnet

@@ -28,22 +28,7 @@
       // Block index-headers are pre-downloaded but lazy mmaped and loaded at query time.
       'blocks-storage.bucket-store.index-header-lazy-loading-enabled': 'true',
       'blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout': '60m',
-
       'blocks-storage.bucket-store.max-chunk-pool-bytes': 12 * 1024 * 1024 * 1024,
-
-      // We should keep a number of idle connections equal to the max "get" concurrency,
-      // in order to avoid re-opening connections continuously (this would be slower
-      // and fill up the conntrack table too).
-      //
-      // The downside of this approach is that we'll end up with an higher number of
-      // active connections to memcached, so we have to make sure connections limit
-      // set in memcached is high enough.
-      'blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency': 100,
-      'blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency': 100,
-      'blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency': 100,
-      'blocks-storage.bucket-store.index-cache.memcached.max-idle-connections': $.store_gateway_args['blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency'],
-      'blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections': $.store_gateway_args['blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency'],
-      'blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections': $.store_gateway_args['blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency'],
     } +
     $.blocks_chunks_caching_config +
     $.blocks_metadata_caching_config +

cortex/tsdb-config.libsonnet

@@ -26,6 +26,14 @@
     cortex_bucket_index_enabled: false,
   },
 
+  // We should keep a number of idle connections equal to the max "get" concurrency,
+  // in order to avoid re-opening connections continuously (this would be slower
+  // and fill up the conntrack table too).
+  //
+  // The downside of this approach is that we'll end up with an higher number of
+  // active connections to memcached, so we have to make sure connections limit
+  // set in memcached is high enough.
+
   blocks_chunks_caching_config::
     (
       if $._config.memcached_index_queries_enabled then {
@@ -36,6 +44,8 @@
         'blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size': '25000',
         'blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency': '50',
         'blocks-storage.bucket-store.index-cache.memcached.max-get-multi-batch-size': '100',
+        'blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency': 100,
+        'blocks-storage.bucket-store.index-cache.memcached.max-idle-connections': self['blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency'],
       } else {}
     ) + (
       if $._config.memcached_chunks_enabled then {
@@ -46,6 +56,8 @@
         'blocks-storage.bucket-store.chunks-cache.memcached.max-async-buffer-size': '25000',
         'blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency': '50',
         'blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-batch-size': '100',
+        'blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency': 100,
+        'blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections': self['blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency'],
       } else {}
     ),
 
@@ -57,6 +69,8 @@
     'blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size': '25000',
     'blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency': '50',
     'blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-batch-size': '100',
+    'blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency': 100,
+    'blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections': self['blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency'],
   } else {},
 
   bucket_index_config:: if $._config.cortex_bucket_index_enabled then {