Merge pull request #8752 from Krishnachaitanyakc/implement/query-exemplar-proxy-strips-external-label-matcher

query: fix exemplar proxy stripping external label matchers in multi-tier topologies

Author: GiedriusS

.mdox.validate.yaml

@@ -61,4 +61,10 @@ validators:
     type: 'ignore'
   - regex: 'codeburst\.io'
     type: 'ignore'
+  # Promtail docs removed after EOL (March 2026).
+  - regex: 'grafana\.com\/docs\/loki\/latest\/clients\/promtail'
+    type: 'ignore'
+  # Frequent DNS/timeout issues from CI.
+  - regex: 'db\.cs\.cmu\.edu'
+    type: 'ignore'
 

CHANGELOG.md

@@ -14,6 +14,7 @@ It is recommend to upgrade the storage components first (Receive, Store, etc.) a
 
 ### Fixed
 
+- [#8702](https://github.com/thanos-io/thanos/issues/8702): Query: Fix exemplar proxy stripping external label matchers in multi-tier query topologies. In Query A → Query B → Sidecar setups, external label matchers are now preserved when forwarding to downstream Query nodes so they can route to the correct stores.
 - [#8726](https://github.com/thanos-io/thanos/pull/8726): *: Bump `thanos-community/grpc-go` fork to fix CVE-2026-33186 (CVSS 9.1), an authorization bypass via malformed `:path` headers that could bypass path-based "deny" rules in `grpc/authz` interceptors.
 - [#8714](https://github.com/thanos-io/thanos/pull/8714): Tracing: Fix `tls_config` fields (`ca_file`, `cert_file`, `key_file`) being silently ignored when using the OTLP gRPC exporter. Previously, deployments using a private CA or mTLS client certificates had to work around this via `OTEL_EXPORTER_OTLP_CERTIFICATE` and related environment variables.
 - [#8128](https://github.com/thanos-io/thanos/issues/8128): Query-Frontend: Fix panic in `AnalyzesMerge` caused by indexing the wrong slice variable, leading to an out-of-range access when merging more than two query analyses.

docs/components/receive.md

@@ -372,7 +372,8 @@ Please see the metric `thanos_receive_forward_delay_seconds` to see if you need
 
 The following formula is used for calculating quorum:
 
-```go mdox-exec="sed -n '1068,1078p' pkg/receive/handler.go"
+```go mdox-exec="sed -n '1067,1078p' pkg/receive/handler.go"
+// writeQuorum returns minimum number of replicas that has to confirm write success before claiming replication success.
 func (h *Handler) writeQuorum() int {
 	// NOTE(GiedriusS): this is here because otherwise RF=2 doesn't make sense as all writes
 	// would need to succeed all the time. Another way to think about it is when migrating

pkg/exemplars/exemplarspb/custom.go

@@ -17,7 +17,8 @@ import (
 // ExemplarStore wraps the ExemplarsClient and contains the info of external labels.
 type ExemplarStore struct {
 	ExemplarsClient
-	LabelSets []labels.Labels
+	LabelSets              []labels.Labels
+	SupportsExternalLabels bool
 }
 
 // UnmarshalJSON implements json.Unmarshaler.

pkg/exemplars/proxy.go

@@ -95,10 +95,13 @@ func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exe
 
 			labelMatchers = labelMatchers[:0]
 			for _, m := range matcherSet {
-				if containsLabelName(m.Name, extLbls) {
+				if !st.SupportsExternalLabels && containsLabelName(m.Name, extLbls) {
 					// If the current matcher matches one external label,
 					// we don't add it to the current metric selector
 					// as Prometheus' Exemplars API cannot handle external labels.
+					// However, if the downstream store supports external labels
+					// (e.g., a Query node), we preserve the matchers so it can
+					// use them for its own routing.
 					continue
 				}
 				labelMatchers = append(labelMatchers, m.String())

pkg/exemplars/proxy_test.go

@@ -379,6 +379,155 @@ func TestProxy(t *testing.T) {
 	}
 }
 
+// testExemplarClientWithQueryCapture extends testExemplarClient to capture the forwarded query.
+type testExemplarClientWithQueryCapture struct {
+	testExemplarClient
+	capturedQuery atomic.String
+}
+
+func (t *testExemplarClientWithQueryCapture) Exemplars(ctx context.Context, in *exemplarspb.ExemplarsRequest, opts ...grpc.CallOption) (exemplarspb.Exemplars_ExemplarsClient, error) {
+	t.capturedQuery.Store(in.Query)
+	return t.testExemplarClient.Exemplars(ctx, in, opts...)
+}
+
+var _ exemplarspb.ExemplarsClient = &testExemplarClientWithQueryCapture{}
+
+func TestProxyExternalLabelsPreservedForQueryStores(t *testing.T) {
+	logger := log.NewLogfmtLogger(os.Stderr)
+
+	// Simulate a multi-tier topology: Query A → [Query B (cluster=A), Sidecar (cluster=B)]
+	// When querying with {cluster="A"}, Query B should receive cluster="A" in its query
+	// (SupportsExternalLabels=true), while a Sidecar would have it stripped.
+
+	queryClient := &testExemplarClientWithQueryCapture{
+		testExemplarClient: testExemplarClient{
+			response: exemplarspb.NewExemplarsResponse(&exemplarspb.ExemplarData{
+				SeriesLabels: labelpb.ZLabelSet{Labels: labelpb.ZLabelsFromPromLabels(labels.FromMap(map[string]string{"__name__": "http_request_duration_bucket"}))},
+				Exemplars:    []*exemplarspb.Exemplar{{Value: 1}},
+			}),
+		},
+	}
+
+	sidecarClient := &testExemplarClientWithQueryCapture{
+		testExemplarClient: testExemplarClient{
+			response: exemplarspb.NewExemplarsResponse(&exemplarspb.ExemplarData{
+				SeriesLabels: labelpb.ZLabelSet{Labels: labelpb.ZLabelsFromPromLabels(labels.FromMap(map[string]string{"__name__": "http_request_duration_bucket"}))},
+				Exemplars:    []*exemplarspb.Exemplar{{Value: 2}},
+			}),
+		},
+	}
+
+	clients := []*exemplarspb.ExemplarStore{
+		{
+			ExemplarsClient:        queryClient,
+			LabelSets:              []labels.Labels{labels.FromMap(map[string]string{"cluster": "A"})},
+			SupportsExternalLabels: true, // This is a Query node.
+		},
+		{
+			ExemplarsClient:        sidecarClient,
+			LabelSets:              []labels.Labels{labels.FromMap(map[string]string{"cluster": "B"})},
+			SupportsExternalLabels: false, // This is a Sidecar.
+		},
+	}
+
+	p := NewProxy(logger, func() []*exemplarspb.ExemplarStore {
+		return clients
+	}, labels.EmptyLabels())
+
+	server := &testExemplarServer{}
+
+	// Query with cluster="A" and namespace="foo".
+	err := p.Exemplars(&exemplarspb.ExemplarsRequest{
+		Query:                   `http_request_duration_bucket{cluster="A", namespace="foo"}`,
+		PartialResponseStrategy: storepb.PartialResponseStrategy_WARN,
+	}, server)
+	testutil.Ok(t, err)
+
+	// Only the Query store (cluster=A) should have been queried.
+	testutil.Equals(t, 1, len(server.responses))
+
+	// Verify the Query node received the full matchers including external label.
+	queryForwarded := queryClient.capturedQuery.Load()
+	testutil.Assert(t, queryForwarded != "", "query store should have been called")
+
+	// The forwarded query to the Query node must contain cluster="A"
+	// because it needs it for its own downstream routing.
+	expr, err := extpromql.ParseExpr(queryForwarded)
+	testutil.Ok(t, err)
+	selectors := parser.ExtractSelectors(expr)
+	testutil.Assert(t, len(selectors) > 0, "expected at least one selector")
+
+	foundCluster := false
+	foundNamespace := false
+	for _, matcherSet := range selectors {
+		for _, m := range matcherSet {
+			if m.Name == "cluster" && m.Value == "A" {
+				foundCluster = true
+			}
+			if m.Name == "namespace" && m.Value == "foo" {
+				foundNamespace = true
+			}
+		}
+	}
+	testutil.Assert(t, foundCluster, "query store should receive cluster matcher for downstream routing")
+	testutil.Assert(t, foundNamespace, "query store should receive namespace matcher")
+
+	// Verify the Sidecar was NOT queried (cluster=B doesn't match cluster="A").
+	sidecarForwarded := sidecarClient.capturedQuery.Load()
+	testutil.Assert(t, sidecarForwarded == "", "sidecar with cluster=B should not be queried for cluster=A request")
+}
+
+func TestProxyExternalLabelsStrippedForSidecarStores(t *testing.T) {
+	logger := log.NewLogfmtLogger(os.Stderr)
+
+	// Verify that the existing behavior of stripping external labels for Sidecars is preserved.
+	sidecarClient := &testExemplarClientWithQueryCapture{
+		testExemplarClient: testExemplarClient{
+			response: exemplarspb.NewExemplarsResponse(&exemplarspb.ExemplarData{
+				SeriesLabels: labelpb.ZLabelSet{Labels: labelpb.ZLabelsFromPromLabels(labels.FromMap(map[string]string{"__name__": "http_request_duration_bucket"}))},
+				Exemplars:    []*exemplarspb.Exemplar{{Value: 1}},
+			}),
+		},
+	}
+
+	clients := []*exemplarspb.ExemplarStore{
+		{
+			ExemplarsClient:        sidecarClient,
+			LabelSets:              []labels.Labels{labels.FromMap(map[string]string{"cluster": "A"})},
+			SupportsExternalLabels: false, // Sidecar.
+		},
+	}
+
+	p := NewProxy(logger, func() []*exemplarspb.ExemplarStore {
+		return clients
+	}, labels.EmptyLabels())
+
+	server := &testExemplarServer{}
+
+	err := p.Exemplars(&exemplarspb.ExemplarsRequest{
+		Query:                   `http_request_duration_bucket{cluster="A", namespace="foo"}`,
+		PartialResponseStrategy: storepb.PartialResponseStrategy_WARN,
+	}, server)
+	testutil.Ok(t, err)
+	testutil.Equals(t, 1, len(server.responses))
+
+	// Verify the Sidecar received the query WITHOUT the external label matcher.
+	sidecarForwarded := sidecarClient.capturedQuery.Load()
+	testutil.Assert(t, sidecarForwarded != "", "sidecar should have been called")
+
+	expr, err := extpromql.ParseExpr(sidecarForwarded)
+	testutil.Ok(t, err)
+	selectors := parser.ExtractSelectors(expr)
+	testutil.Assert(t, len(selectors) > 0, "expected at least one selector")
+
+	for _, matcherSet := range selectors {
+		for _, m := range matcherSet {
+			testutil.Assert(t, m.Name != "cluster",
+				"sidecar should NOT receive external label matcher cluster, but got: %s", m.String())
+		}
+	}
+}
+
 // TestProxyDataRace find the concurrent data race bug ( go test -race -run TestProxyDataRace -v ).
 func TestProxyDataRace(t *testing.T) {
 	logger := log.NewLogfmtLogger(os.Stderr)

pkg/query/endpointset.go

@@ -600,8 +600,9 @@ func (e *EndpointSet) GetExemplarsStores() []*exemplarspb.ExemplarStore {
 	for _, er := range endpoints {
 		if er.HasExemplarsAPI() {
 			exemplarStores = append(exemplarStores, &exemplarspb.ExemplarStore{
-				ExemplarsClient: exemplarspb.NewExemplarsClient(er.cc),
-				LabelSets:       labelpb.ZLabelSetsToPromLabelSets(er.metadata.LabelSets...),
+				ExemplarsClient:        exemplarspb.NewExemplarsClient(er.cc),
+				LabelSets:              labelpb.ZLabelSetsToPromLabelSets(er.metadata.LabelSets...),
+				SupportsExternalLabels: er.ComponentType() == component.Query,
 			})
 		}
 	}