[Query-Frontend] Add dynamic query vertical sharding (#6678)

* refactor dynamic query splitting to reduce complexity

Signed-off-by: Ahmed Hassan <[email protected]>

* add query dynamic vertical sharding

Signed-off-by: Ahmed Hassan <[email protected]>

* fix unit tests for dynamic vertical sharding

Signed-off-by: Ahmed Hassan <[email protected]>

* store dynamic vertical shard size in context

Signed-off-by: Ahmed Hassan <[email protected]>

* refactoring dynamicIntervalFn

Signed-off-by: Ahmed Hassan <[email protected]>

* add separate vertical sharding tests

Signed-off-by: Ahmed Hassan <[email protected]>

* fix formatting

Signed-off-by: Ahmed Hassan <[email protected]>

* add feature flag for dynamic vertical sharding

Signed-off-by: Ahmed Hassan <[email protected]>

* update doc

Signed-off-by: Ahmed Hassan <[email protected]>

* rerun tests

Signed-off-by: Ahmed Hassan <[email protected]>

---------

Signed-off-by: Ahmed Hassan <[email protected]>

Author: afhassan

docs/configuration/config-file-reference.md

@@ -4354,6 +4354,11 @@ dynamic_query_splits:
   # CLI flag: -querier.max-fetched-data-duration-per-query
   [max_fetched_data_duration_per_query: <duration> | default = 0s]
 
+  # [EXPERIMENTAL] Dynamically adjust vertical shard size to maximize the total
+  # combined number of query shards and splits.
+  # CLI flag: -querier.enable-dynamic-vertical-sharding
+  [enable_dynamic_vertical_sharding: <boolean> | default = false]
+
 # Mutate incoming queries to align their start and end with their step.
 # CLI flag: -querier.align-querier-with-step
 [align_queries_with_step: <boolean> | default = false]

pkg/querier/tripperware/queryrange/query_range_middlewares.go

@@ -81,12 +81,14 @@ func (cfg *Config) Validate(qCfg querier.Config) error {
 type DynamicQuerySplitsConfig struct {
 	MaxShardsPerQuery              int           `yaml:"max_shards_per_query"`
 	MaxFetchedDataDurationPerQuery time.Duration `yaml:"max_fetched_data_duration_per_query"`
+	EnableDynamicVerticalSharding  bool          `yaml:"enable_dynamic_vertical_sharding"`
 }
 
 // RegisterFlags registers flags foy dynamic query splits
 func (cfg *DynamicQuerySplitsConfig) RegisterFlags(f *flag.FlagSet) {
 	f.IntVar(&cfg.MaxShardsPerQuery, "querier.max-shards-per-query", 0, "[EXPERIMENTAL] Maximum number of shards for a query, 0 disables it. Dynamically uses a multiple of split interval to maintain a total number of shards below the set value. If vertical sharding is enabled for a query, the combined total number of interval splits and vertical shards is kept below this value.")
 	f.DurationVar(&cfg.MaxFetchedDataDurationPerQuery, "querier.max-fetched-data-duration-per-query", 0, "[EXPERIMENTAL] Max total duration of data fetched from storage by all query shards, 0 disables it. Dynamically uses a multiple of split interval to maintain a total fetched duration of data lower than the value set. It takes into account additional duration fetched by matrix selectors and subqueries.")
+	f.BoolVar(&cfg.EnableDynamicVerticalSharding, "querier.enable-dynamic-vertical-sharding", false, "[EXPERIMENTAL] Dynamically adjust vertical shard size to maximize the total combined number of query shards and splits.")
 }
 
 // Middlewares returns list of middlewares that should be applied for range query.