From 4254d409dfd5c0c40a02faffc43987d351d65fc7 Mon Sep 17 00:00:00 2001 From: Essam Eldaly Date: Tue, 25 Nov 2025 13:18:11 -0800 Subject: [PATCH 1/5] Add query priority for all operations Signed-off-by: Essam Eldaly --- pkg/querier/tripperware/query_attribute_matcher.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/querier/tripperware/query_attribute_matcher.go b/pkg/querier/tripperware/query_attribute_matcher.go index 36e38103952..697de64269e 100644 --- a/pkg/querier/tripperware/query_attribute_matcher.go +++ b/pkg/querier/tripperware/query_attribute_matcher.go @@ -22,6 +22,7 @@ func rejectQueryOrSetPriority(r *http.Request, now time.Time, lookbackDelta time return nil } op := getOperation(r) + reqStats := stats.FromContext(r.Context()) if op == "query" || op == "query_range" { query := r.FormValue("query") @@ -40,7 +41,6 @@ func rejectQueryOrSetPriority(r *http.Request, now time.Time, lookbackDelta time } } - reqStats := stats.FromContext(r.Context()) reqStats.SetDataSelectMaxTime(maxTime) reqStats.SetDataSelectMinTime(minTime) @@ -55,6 +55,16 @@ func rejectQueryOrSetPriority(r *http.Request, now time.Time, lookbackDelta time } reqStats.SetPriority(queryPriority.DefaultPriority) } + } else { + if queryPriority := limits.QueryPriority(userStr); queryPriority.Enabled && len(queryPriority.Priorities) != 0 { + for _, priority := range queryPriority.Priorities { + for _, attribute := range priority.QueryAttributes { + if matchAttributeForMetadataQuery(attribute, op, r, now) { + reqStats.SetPriority(priority.Priority) + } + } + } + } } if queryReject := limits.QueryRejection(userStr); queryReject.Enabled && (op == "series" || op == "labels" || op == "label_values") { @@ -81,6 +91,8 @@ func getOperation(r *http.Request) string { return "labels" case strings.HasSuffix(r.URL.Path, "/values"): return "label_values" + case strings.HasSuffix(r.URL.Path, "/metadata"): + return "metadata" default: return "other" } From 7878a8c0906c5b3e0ba8c3acabfde0285cd7e10c Mon Sep 17 00:00:00 2001 From: Essam Eldaly Date: Tue, 25 Nov 2025 13:26:50 -0800 Subject: [PATCH 2/5] Move query reject to before query priority Signed-off-by: Essam Eldaly --- .../tripperware/query_attribute_matcher.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/querier/tripperware/query_attribute_matcher.go b/pkg/querier/tripperware/query_attribute_matcher.go index 697de64269e..ea3c80577af 100644 --- a/pkg/querier/tripperware/query_attribute_matcher.go +++ b/pkg/querier/tripperware/query_attribute_matcher.go @@ -56,6 +56,15 @@ func rejectQueryOrSetPriority(r *http.Request, now time.Time, lookbackDelta time reqStats.SetPriority(queryPriority.DefaultPriority) } } else { + if queryReject := limits.QueryRejection(userStr); queryReject.Enabled && (op == "series" || op == "labels" || op == "label_values") { + for _, attribute := range queryReject.QueryAttributes { + if matchAttributeForMetadataQuery(attribute, op, r, now) { + rejectedQueriesPerTenant.WithLabelValues(op, userStr).Inc() + return httpgrpc.Errorf(http.StatusUnprocessableEntity, QueryRejectErrorMessage) + } + } + } + if queryPriority := limits.QueryPriority(userStr); queryPriority.Enabled && len(queryPriority.Priorities) != 0 { for _, priority := range queryPriority.Priorities { for _, attribute := range priority.QueryAttributes { @@ -67,15 +76,6 @@ func rejectQueryOrSetPriority(r *http.Request, now time.Time, lookbackDelta time } } - if queryReject := limits.QueryRejection(userStr); queryReject.Enabled && (op == "series" || op == "labels" || op == "label_values") { - for _, attribute := range queryReject.QueryAttributes { - if matchAttributeForMetadataQuery(attribute, op, r, now) { - rejectedQueriesPerTenant.WithLabelValues(op, userStr).Inc() - return httpgrpc.Errorf(http.StatusUnprocessableEntity, QueryRejectErrorMessage) - } - } - } - return nil } From d590128265b7c59f5e275d4c1efe79a5cdc04345 Mon Sep 17 00:00:00 2001 From: Essam Eldaly Date: Tue, 25 Nov 2025 13:59:07 -0800 Subject: [PATCH 3/5] Update tests and changelog Signed-off-by: Essam Eldaly --- CHANGELOG.md | 1 + .../query_attribute_matcher_test.go | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8203b37c23b..c09fe5f5df3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * [ENHANCEMENT] Distributor: Add a label references validation for remote write v2 request. #7074 * [ENHANCEMENT] Distributor: Add count, spans, and buckets validations for native histogram. #7072 * [ENHANCEMENT] Ruler: Add DecodingConcurrency config flag for Thanos Engine. #7118 +* [ENHANCEMENT] Query Frontend: Add query priority based on operation. #7127 * [BUGFIX] Ring: Change DynamoDB KV to retry indefinitely for WatchKey. #7088 * [BUGFIX] Ruler: Add XFunctions validation support. #7111 diff --git a/pkg/querier/tripperware/query_attribute_matcher_test.go b/pkg/querier/tripperware/query_attribute_matcher_test.go index cbd1f949f84..31db4e30e55 100644 --- a/pkg/querier/tripperware/query_attribute_matcher_test.go +++ b/pkg/querier/tripperware/query_attribute_matcher_test.go @@ -706,3 +706,45 @@ func Test_matchAttributeForExpressionQueryShouldMatchUserAgentRegex(t *testing.T } } + +func Test_rejectQueryOrSetPriorityShouldMatchOnApiType(t *testing.T) { + + limits := mockLimits{queryPriority: validation.QueryPriority{ + Priorities: []validation.PriorityDef{ + { + Priority: 7, + QueryAttributes: []validation.QueryAttribute{ + { + ApiType: "metadata", + }, + }, + }, + }, + }, + } + + type testCase struct { + path string + expectedPriority int64 + } + + tests := map[string]testCase{ + "should set priority based on api type": { + path: "/api/v1/targets/metadata?limit=1", + expectedPriority: 7, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + req, err := http.NewRequest("GET", testData.path, http.NoBody) + require.NoError(t, err) + reqStats, ctx := stats.ContextWithEmptyStats(context.Background()) + req = req.WithContext(ctx) + limits.queryPriority.Enabled = true + resultErr := rejectQueryOrSetPriority(req, time.Now(), time.Duration(1), limits, "", rejectedQueriesPerTenant) + assert.NoError(t, resultErr) + assert.Equal(t, testData.expectedPriority, reqStats.Priority) + }) + } +} From b374909b1da1778b57f054c42a41647d5a38d8dc Mon Sep 17 00:00:00 2001 From: Essam Eldaly Date: Tue, 25 Nov 2025 14:27:23 -0800 Subject: [PATCH 4/5] Fix old test to support change Signed-off-by: Essam Eldaly --- CHANGELOG.md | 2 +- pkg/querier/tripperware/query_attribute_matcher_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c09fe5f5df3..9bd97eb2fb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ * [ENHANCEMENT] Distributor: Add a label references validation for remote write v2 request. #7074 * [ENHANCEMENT] Distributor: Add count, spans, and buckets validations for native histogram. #7072 * [ENHANCEMENT] Ruler: Add DecodingConcurrency config flag for Thanos Engine. #7118 -* [ENHANCEMENT] Query Frontend: Add query priority based on operation. #7127 +* [ENHANCEMENT] Query Frontend: Add query priority based on operation. #7128 * [BUGFIX] Ring: Change DynamoDB KV to retry indefinitely for WatchKey. #7088 * [BUGFIX] Ruler: Add XFunctions validation support. #7111 diff --git a/pkg/querier/tripperware/query_attribute_matcher_test.go b/pkg/querier/tripperware/query_attribute_matcher_test.go index 31db4e30e55..5b34df16427 100644 --- a/pkg/querier/tripperware/query_attribute_matcher_test.go +++ b/pkg/querier/tripperware/query_attribute_matcher_test.go @@ -59,10 +59,11 @@ func Test_rejectQueryOrSetPriorityShouldReturnDefaultPriorityIfNotEnabledOrInval path: "/api/v1/query?time=1536716898&query=", expectedError: httpgrpc.Errorf(http.StatusBadRequest, "unknown position: parse error: no expression found in input"), }, - "should miss if it's metadata query and only priority is enabled": { + "should set priority if it's metadata query and only priority is enabled": { queryPriorityEnabled: true, queryRejectionEnabled: false, path: "/api/v1/labels?match[]", + expectedPriority: 1, }, "should set priority if regex match and rejection disabled": { queryPriorityEnabled: true, From b9026e9095d40086e4aa4e4881f728c6d254eff8 Mon Sep 17 00:00:00 2001 From: Essam Eldaly Date: Tue, 25 Nov 2025 15:46:58 -0800 Subject: [PATCH 5/5] Prioritize top most priority to be consistent with query priorities Signed-off-by: Essam Eldaly --- pkg/querier/tripperware/query_attribute_matcher.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/querier/tripperware/query_attribute_matcher.go b/pkg/querier/tripperware/query_attribute_matcher.go index ea3c80577af..7e0990162bb 100644 --- a/pkg/querier/tripperware/query_attribute_matcher.go +++ b/pkg/querier/tripperware/query_attribute_matcher.go @@ -70,6 +70,7 @@ func rejectQueryOrSetPriority(r *http.Request, now time.Time, lookbackDelta time for _, attribute := range priority.QueryAttributes { if matchAttributeForMetadataQuery(attribute, op, r, now) { reqStats.SetPriority(priority.Priority) + break } } }