Merge remote-tracking branch 'upstream/master' into merge-release-1.19-to-master

Signed-off-by: Charlie Le <[email protected]>

Author: CharlieTLe

.github/workflows/scorecards.yml

@@ -64,6 +64,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard (optional).
       # Commenting out will disable upload of results to your repo's Code Scanning dashboard
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
+        uses: github/codeql-action/upload-sarif@dd746615b3b9d728a6a37ca2045b68ca76d4841a # v3.28.8
         with:
           sarif_file: results.sarif

.github/workflows/test-build-deploy.yml

@@ -15,7 +15,7 @@ on:
 
 jobs:
   lint:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     container:
       image: quay.io/cortexproject/build-image:master-03a8f8c98
     steps:
@@ -44,7 +44,7 @@ jobs:
         run: make BUILD_IN_CONTAINER=false check-white-noise
 
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     container:
       image: quay.io/cortexproject/build-image:master-03a8f8c98
     steps:
@@ -62,7 +62,7 @@ jobs:
       - name: Run Tests
         run: make BUILD_IN_CONTAINER=false test
   test-no-race:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     container:
       image: quay.io/cortexproject/build-image:master-03a8f8c98
     steps:
@@ -82,7 +82,7 @@ jobs:
 
   security:
     name: CodeQL
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     permissions:
       actions: read
       contents: read
@@ -93,19 +93,19 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
+        uses: github/codeql-action/init@dd746615b3b9d728a6a37ca2045b68ca76d4841a # v3.28.8
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
+        uses: github/codeql-action/autobuild@dd746615b3b9d728a6a37ca2045b68ca76d4841a # v3.28.8
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
+        uses: github/codeql-action/analyze@dd746615b3b9d728a6a37ca2045b68ca76d4841a # v3.28.8
 
 
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     container:
       image: quay.io/cortexproject/build-image:master-03a8f8c98
     steps:
@@ -150,7 +150,7 @@ jobs:
 
   integration:
     needs: build
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -164,7 +164,7 @@ jobs:
           - integration_query_fuzz
     steps:
       - name: Upgrade golang
-        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
+        uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
         with:
           go-version: 1.23.2
       - name: Checkout Repo
@@ -230,7 +230,7 @@ jobs:
 
   integration-configs-db:
     needs: build
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout Repo
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -252,7 +252,7 @@ jobs:
   deploy_website:
     needs: [build, test]
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     container:
       image: quay.io/cortexproject/build-image:master-03a8f8c98
     steps:
@@ -294,7 +294,7 @@ jobs:
   deploy:
     needs: [build, test, lint, integration, integration-configs-db]
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     container:
       image: quay.io/cortexproject/build-image:master-03a8f8c98
     steps:

CHANGELOG.md

@@ -1,6 +1,23 @@
 # Changelog
 
 ## master / unreleased
+* [CHANGE] StoreGateway/Alertmanager: Add default 5s connection timeout on client. #6603
+* [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458
+* [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526
+* [FEATURE] Update prometheus alertmanager version to v0.28.0 and add new integration msteamsv2, jira, and rocketchat. #6590
+* [ENHANCEMENT] Alertmanager: Add new limits `-alertmanager.max-silences-count` and `-alertmanager.max-silences-size-bytes` for limiting silences per tenant. #6605
+* [ENHANCEMENT] Add `compactor.auto-forget-delay` for compactor to auto forget compactors after X minutes without heartbeat. #6533
+* [ENHANCEMENT] StoreGateway: Emit more histogram buckets on the `cortex_querier_storegateway_refetches_per_query` metric. #6570
+* [ENHANCEMENT] Querier: Apply bytes limiter to LabelNames and LabelValuesForLabelNames. #6568
+* [ENHANCEMENT] Query Frontend: Add a `too_many_tenants` reason label value to `cortex_rejected_queries_total` metric to track the rejected query count due to the # of tenant limits. #6569
+* [ENHANCEMENT] Alertmanager: Add receiver validations for msteamsv2 and rocketchat. #6606
+* [BUGFIX] Ingester: Avoid error or early throttling when READONLY ingesters are present in the ring #6517
+* [BUGFIX] Ingester: Fix labelset data race condition. #6573
+* [BUGFIX] Compactor: Cleaner should not put deletion marker for blocks with no-compact marker. #6576
+* [BUGFIX] Compactor: Cleaner would delete bucket index when there is no block in bucket store. #6577
+* [BUGFIX] Querier: Fix marshal native histogram with empty bucket when protobuf codec is enabled. #6595
+* [BUGFIX] Query Frontend: Fix samples scanned and peak samples query stats when query hits results cache. #6591
+* [BUGFIX] Query Frontend: Fix panic caused by nil pointer dereference. #6609
 
 ## 1.19.0 in progress
 
@@ -14,15 +31,6 @@
 * [CHANGE] Enable Compactor and Alertmanager in target all. #6204
 * [CHANGE] Update the `cortex_ingester_inflight_push_requests` metric to represent the maximum number of inflight requests recorded in the last minute. #6437
 * [CHANGE] gRPC Client: Expose connection timeout and set default to value to 5s. #6523
-* [CHANGE] Enable analysis on Thanos PromQL engine #6472
-* [CHANGE] Log grafana headers along with query request before query execution #6391
-* [CHANGE] Log query requests in QFE before query execution #6390
-* [CHANGE] Log when a request starts running in querier #6525
-* [CHANGE] Remove openstack swfit experimental in blocks storage #6322
-* [CHANGE] Remove openstack swift experimental #6316
-* [CHANGE] Replace `cespare/xxhash` with `cespare/xxhash/v2` #6467
-* [CHANGE] Stop using global instant query codec #6328
-* [CHANGE] Unify query frontend instant and range protos #6180
 * [FEATURE] Ruler: Add an experimental flag `-ruler.query-response-format` to retrieve query response as a proto format. #6345
 * [FEATURE] Ruler: Pagination support for List Rules API. #6299
 * [FEATURE] Query Frontend/Querier: Add protobuf codec `-api.querier-default-codec` and the option to choose response compression type `-querier.response-compression`. #5527
@@ -35,13 +43,6 @@
 * [FEATURE] Query Frontend: Support a metadata federated query when `-tenant-federation.enabled=true`. #6461
 * [FEATURE] Query Frontend: Support an exemplar federated query when `-tenant-federation.enabled=true`. #6455
 * [FEATURE] Ingester/StoreGateway: Add support for cache regex query matchers via `-ingester.matchers-cache-max-items` and `-blocks-storage.bucket-store.matchers-cache-max-items`. #6477 #6491
-* [FEATURE] Add reason why the key was evicted in the `cortex_ingester_expanded_postings_cache_evicts` metric #6318
-* [FEATURE] Create feature flag to switch between current shuffle sharding group planner and partition compaction group planner #6141
-* [FEATURE] Hook up partition compaction end to end implementation #6510
-* [FEATURE] Implement partition compaction grouper #6172
-* [FEATURE] Implement partition compaction planner #6469
-* [FEATURE] Make LivenessCheck Timeout Configurable #6227
-* [FEATURE] Querier: Add day range limit for LabelNames and LabelValues #6233
 * [ENHANCEMENT] Query Frontend: Add more operation label values to the `cortex_query_frontend_queries_total` metric. #6519
 * [ENHANCEMENT] Query Frontend: Add a `source` label to query stat metrics. #6470
 * [ENHANCEMENT] Query Frontend: Add a flag `-tenant-federation.max-tenant` to limit the number of tenants for federated query. #6493
@@ -82,50 +83,14 @@
 * [ENHANCEMENT] Distributor: Added `cortex_distributor_received_samples_per_labelset_total` metric to calculate ingestion rate per label set. #6443
 * [ENHANCEMENT] Added metric name in limiter per-metric exceeded errors. #6416
 * [ENHANCEMENT] StoreGateway: Added `cortex_bucket_store_indexheader_load_duration_seconds` and `cortex_bucket_store_indexheader_download_duration_seconds` metrics for time of downloading and loading index header files. #6445
-* [ENHANCEMENT] Add cleaner logic to clean partition compaction blocks and related files #6507
-* [ENHANCEMENT] Add support for native histograms in querier protobuf codec #6368
-* [ENHANCEMENT] Add timeout on lifecycler heartbeat #6212
-* [ENHANCEMENT] Added UserReplicaGroupMetrics #6463
 * [ENHANCEMENT] Blocks Storage: Allow use of non-dualstack endpoints for S3 blocks storage via `-blocks-storage.s3.disable-dualstack`. #6522
-* [ENHANCEMENT] Discarded samples per labelset metrics for throttle by labelset #6492
-* [ENHANCEMENT] Expanded Postings Cache can cache results without the nearly created series under high load. #6417
-* [ENHANCEMENT] Ingester: Disable chunk trimming. #6270
-* [ENHANCEMENT] Improve consistency check warn log #6366
-* [ENHANCEMENT] Improve streaming on MetricsForLabelMatchersStream method #6436
-* [ENHANCEMENT] Improve validation metrics for discarded samples and exemplars #6218
-* [ENHANCEMENT] Query Frontend: add new field for dense native histogram format #6199
-* [ENHANCEMENT] Return 503 on hitting distributor instance limits #6387
-* [ENHANCEMENT] Reusing Batch Iterators #6403
-* [ENHANCEMENT] Reusing the grpc client to peform healthcheck #6260
-* [ENHANCEMENT] Store Gateway: Add pre add block ownership check #6483
-* [ENHANCEMENT] Use slice pooling to populate the query stream response #6466
-* [ENHANCEMENT] Using a single seed array for expanded postings cache on ingesters #6365
 * [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224
 * [BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. #6326
-* [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled. #6271
+* [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled.  #6271
 * [BUGFIX] Ingester: Fix regression on usage of cortex_ingester_queried_chunks. #6398
 * [BUGFIX] Ingester: Fix possible race condition when `active series per LabelSet` is configured. #6409
 * [BUGFIX] Query Frontend: Fix @ modifier not being applied correctly on sub queries. #6450
 * [BUGFIX] Cortex Redis flags with multiple dots #6476
-* [BUGFIX] Bug fix on JSON Tag #6339
-* [BUGFIX] Calculate # of concurrency only once at the runner #6506
-* [BUGFIX] Clean up ingester per labelset metrics #6439
-* [BUGFIX] Cleanup dangling request queue metrics #6433
-* [BUGFIX] Fix BenchmarkDistributor_Push benchmark #6309
-* [BUGFIX] Fix data race on expanded postings Cache #6369
-* [BUGFIX] Fix lazy postings merge bug #6415
-* [BUGFIX] Fix race on chunks multilevel cache + Optimize to avoid refetching already found keys. #6312
-* [BUGFIX] Fix race on the string interning #6408
-* [BUGFIX] Fix race that can cause nil reference when using expanded postings #6518
-* [BUGFIX] Fix regression of query range result cache unable to parse old cached results #6196
-* [BUGFIX] Fix typo in usage message for querier.split-queries-by-interval flag #6305
-* [BUGFIX] Fix: PostingCache promise should fetch data only once #6314
-* [BUGFIX] Fix: fix slice init length #6237
-* [BUGFIX] Fixed bug that blocks cannot be fully deleted from TSDB #6231
-* [BUGFIX] Fixed ingester ReadOnly state related bugs #6208
-* [BUGFIX] Preserve ingester state on restart #6301
-* [BUGFIX] Purge expired postings cache items due inactivity #6502
-* [BUGFIX] Util: Check context every N iterations #6250
 
 ## 1.18.1 2024-10-14
 
@@ -1225,7 +1190,7 @@ Note the blocks storage compactor runs a migration task at startup in this versi
 * [ENHANCEMENT] Improve performance of QueryStream() in ingesters. #3177
 * [ENHANCEMENT] Modules included in "All" target are now visible in output of `-modules` CLI flag. #3155
 * [ENHANCEMENT] Added `/debug/fgprof` endpoint to debug running Cortex process using `fgprof`. This adds up to the existing `/debug/...` endpoints. #3131
-* [ENHANCEMENT] Blocks storage: optimised `/api/v1/series` for blocks storage. #2976
+* [ENHANCEMENT] Blocks storage: optimised `/api/v1/series` for blocks storage. (#2976)
 * [BUGFIX] Ruler: when loading rules from "local" storage, check for directory after resolving symlink. #3137
 * [BUGFIX] Query-frontend: Fixed rounding for incoming query timestamps, to be 100% Prometheus compatible. #2990
 * [BUGFIX] Querier: Merge results from chunks and blocks ingesters when using streaming of results. #3013

MAINTAINERS.md

@@ -13,5 +13,6 @@
 
 | Name            | Email                      | GitHub          | Company             |
 |-----------------|----------------------------|-----------------|---------------------|
-| Sungjin Lee     | [email protected]       | @SungJin1212    | KakaoEnterprise     |
 | Anand Rajagopal | [email protected] | @rajagopalanand | Amazon Web Services |
+| Sungjin Lee     | [email protected]       | @SungJin1212    | KakaoEnterprise     |
+| Daniel Sabsay   | [email protected]          | @dsabsay        | Adobe               |

docs/blocks-storage/compactor.md

@@ -225,6 +225,10 @@ compactor:
         # CLI flag: -compactor.ring.dynamodb.max-cas-retries
         [max_cas_retries: <int> | default = 10]
 
+        # Timeout of dynamoDbClient requests. Default is 2m.
+        # CLI flag: -compactor.ring.dynamodb.timeout
+        [timeout: <duration> | default = 2m]
+
       # The consul_config configures the consul client.
       # The CLI flags prefix for this block config is: compactor.ring
       [consul: <consul_config>]
@@ -259,6 +263,11 @@ compactor:
     # CLI flag: -compactor.ring.heartbeat-timeout
     [heartbeat_timeout: <duration> | default = 1m]
 
+    # Time since last heartbeat before compactor will be removed from ring. 0 to
+    # disable
+    # CLI flag: -compactor.auto-forget-delay
+    [auto_forget_delay: <duration> | default = 2m]
+
     # Minimum time to wait for ring stability at startup. 0 to disable.
     # CLI flag: -compactor.ring.wait-stability-min-duration
     [wait_stability_min_duration: <duration> | default = 1m]

docs/blocks-storage/querier.md

@@ -222,6 +222,11 @@ querier:
       # CLI flag: -querier.store-gateway-client.healthcheck.timeout
       [timeout: <duration> | default = 1s]
 
+    # The maximum amount of time to establish a connection. A value of 0 means
+    # using default gRPC client connect timeout 5s.
+    # CLI flag: -querier.store-gateway-client.connect-timeout
+    [connect_timeout: <duration> | default = 5s]
+
   # If enabled, store gateway query stats will be logged using `info` log level.
   # CLI flag: -querier.store-gateway-query-stats-enabled
   [store_gateway_query_stats: <boolean> | default = true]

docs/blocks-storage/store-gateway.md

@@ -240,6 +240,10 @@ store_gateway:
         # CLI flag: -store-gateway.sharding-ring.dynamodb.max-cas-retries
         [max_cas_retries: <int> | default = 10]
 
+        # Timeout of dynamoDbClient requests. Default is 2m.
+        # CLI flag: -store-gateway.sharding-ring.dynamodb.timeout
+        [timeout: <duration> | default = 2m]
+
       # The consul_config configures the consul client.
       # The CLI flags prefix for this block config is:
       # store-gateway.sharding-ring