[8.9] Enable all remaining metric aggregations on counters (#97974) (#98294)

* Enable all remaining metric aggregations on counters (#97974)

Here we enable aggregations previously not allowed on fields of type counter.
The decision of enabling such aggregations even if the result is "meaningless"
for counters has been taken to favour TSDB adoption.

Aggregations now allowed, other than the existing ones, include:
* avg
* box plot
* cardinality
* extended stats
* median absolute deviation
* percentile ranks
* percentiles
* stats
* sum
* value count

I included tests for the weighted average and matrix stats aggregations too.

Resolves #97882

(cherry picked from commit d0b2f65)

# Conflicts:
#	rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/120_counter_fields.yml

* fix: skip versions up to and including 8.9.0

Author: salvatore-campagna

docs/reference/data-streams/tsds.asciidoc

@@ -155,7 +155,7 @@ of aggregations (for example `sum`) compute results that don't make sense for a
 +
 Only numeric and `aggregate_metric_double` fields support the `counter` metric type.
 
-NOTE: Due to the cumulative nature of counter fields, only the following aggregations are allowed with the `counter` field: `rate`, `histogram`, `range`, `min`, `max`, `top_metrics` and `variable_width_histogram`.
+NOTE: Due to the cumulative nature of counter fields, the following aggregations are supported and expected to provide meaningful results with the `counter` field: `rate`, `histogram`, `range`, `min`, `max`, `top_metrics` and `variable_width_histogram`. In order to prevent issues with existing integrations and custom dashboards, we also allow the following aggregations, even if the result might be meaningless on counters: `avg`, `box plot`, `cardinality`, `extended stats`, `median absolute deviation`, `percentile ranks`, `percentiles`, `stats`, `sum` and `value count`.
 
 // tag::time-series-metric-gauge[]
 `gauge`:: A metric that represents a single numeric that can arbitrarily increase or decrease. For example, a temperature or

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/120_counter_fields.yml

@@ -1,53 +1,324 @@
 ---
-"avg aggregation on counter field":
+setup:
   - skip:
-      version: " - 8.6.99"
-      reason: "counter field support added in 8.7"
+      version: " - 8.9.0"
+      reason: "additional counter field aggregations back-ported from 8.10 to 8.9.1"
+      features: close_to
 
   - do:
       indices.create:
-        index: myindex1
-        body:
-          mappings:
-            properties:
-              counter_field:
-                type : long
-                time_series_metric: counter
-  - do:
-      indices.create:
-        index: myindex2
+        index: test_counter
         body:
           settings:
             index:
               mode: time_series
-              routing_path: [ keyword_field ]
+              routing_path: [ key ]
               time_series:
-                start_time: 2023-01-01T00:00:00Z
-                end_time: 2024-01-01T00:00:00Z
+                start_time: 2021-01-01T00:00:00Z
+                end_time: 2021-01-31T00:00:00Z
+              number_of_shards: 1
           mappings:
             properties:
-              keyword_field:
+              "@timestamp":
+                type: date
+              key:
                 type: keyword
                 time_series_dimension: true
-              counter_field:
-                type : long
+              counter:
+                type: long
                 time_series_metric: counter
+              weight:
+                type: integer
+
+  - do:
+      bulk:
+        index: test_counter
+        refresh: true
+        body:
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10, "weight": 2 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20, "weight": 1 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22, "weight": 2 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28, "weight": 1 }'
+
+---
+"avg":
   - do:
       search:
-        index: myindex1
+        index: test_counter
         body:
           aggs:
-            the_counter_avg:
+            counter_avg:
               avg:
-                field: counter_field
-  - match: { aggregations.the_counter_avg.value: null }
+                field: counter
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - close_to: { aggregations.counter_avg.value: { value: 20.00, error: 0.01 } }
 
+---
+"cardinality":
   - do:
-      catch: /Field \[counter_field\] of type \[long\] is not supported for aggregation \[avg\]/
       search:
-        index: myindex2
+        index: test_counter
         body:
           aggs:
-            the_counter_avg:
-              avg:
-                field: counter_field
+            counter_cardinality:
+              cardinality:
+                field: counter
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - match: { aggregations.counter_cardinality.value:  4 }
+
+---
+"extended stats":
+  - do:
+      search:
+        index: test_counter
+        body:
+          aggs:
+            counter_extended_stats:
+              extended_stats:
+                field: counter
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - match: { aggregations.counter_extended_stats.count: 4 }
+  - close_to: { aggregations.counter_extended_stats.min: { value: 10.00, error: 0.01 } }
+  - close_to: { aggregations.counter_extended_stats.max: { value: 28.00, error: 0.01 } }
+  - close_to: { aggregations.counter_extended_stats.avg: { value: 20.00, error: 0.01 } }
+  - close_to: { aggregations.counter_extended_stats.sum: { value: 80.00, error: 0.01 } }
+  - close_to: { aggregations.counter_extended_stats.sum_of_squares: { value: 1768.00, error: 0.01 } }
+  - close_to: { aggregations.counter_extended_stats.std_deviation: { value: 6.48, error: 0.01 } }
+  - close_to: { aggregations.counter_extended_stats.std_deviation_bounds.upper: { value: 32.96, error: 0.01 } }
+  - close_to: { aggregations.counter_extended_stats.std_deviation_bounds.lower: { value: 7.03, error: 0.01 } }
+
+---
+"median absolute deviation":
+  - do:
+      search:
+        index: test_counter
+        body:
+          aggs:
+            mad_counter:
+              median_absolute_deviation:
+                field: counter
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - close_to: { aggregations.mad_counter.value: { value: 4.00, error: 0.01 } }
+
+---
+"percentile ranks hdr":
+  - do:
+      search:
+        index: test_counter
+        body:
+          aggs:
+            counter_percentile_ranks:
+              percentile_ranks:
+                field: counter
+                values: [50, 90]
+                keyed: false
+                hdr:
+                  number_of_significant_value_digits: 2
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - close_to: { aggregations.counter_percentile_ranks.values.0.value: { value: 100.00, error: 0.01 } }
+  - close_to: { aggregations.counter_percentile_ranks.values.1.value: { value: 100.00, error: 0.01 } }
+
+---
+"percentile ranks tdigest":
+  - do:
+      search:
+        index: test_counter
+        body:
+          aggs:
+            counter_percentiles:
+              percentiles:
+                field: counter
+                keyed: false
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - close_to: { aggregations.counter_percentiles.values.0.value: { value: 10.30, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.1.value: { value: 11.50, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.2.value: { value: 17.50, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.3.value: { value: 21.00, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.4.value: { value: 23.50, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.5.value: { value: 27.10, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.6.value: { value: 27.82, error: 0.01 } }
+
+---
+"percentiles hdr":
+  - do:
+      search:
+        index: test_counter
+        body:
+          aggs:
+            counter_percentiles:
+              percentiles:
+                field: counter
+                keyed: false
+                hdr:
+                  number_of_significant_value_digits: 2
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - close_to: { aggregations.counter_percentiles.values.0.value: { value: 10.00, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.1.value: { value: 10.00, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.2.value: { value: 10.00, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.3.value: { value: 20.06, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.4.value: { value: 22.06, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.5.value: { value: 28.06, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.6.value: { value: 28.06, error: 0.01 } }
+
+---
+"percentiles tdigest":
+  - do:
+      bulk:
+        index: test_counter
+        refresh: true
+        body:
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }'
+
+  - do:
+      search:
+        index: test_counter
+        body:
+          aggs:
+            counter_percentiles:
+              percentiles:
+                field: counter
+                keyed: false
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - close_to: { aggregations.counter_percentiles.values.0.value: { value: 10.30, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.1.value: { value: 11.50, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.2.value: { value: 17.50, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.3.value: { value: 21.00, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.4.value: { value: 23.50, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.5.value: { value: 27.10, error: 0.01 } }
+  - close_to: { aggregations.counter_percentiles.values.6.value: { value: 27.82, error: 0.01 } }
+
+---
+"stats":
+  - do:
+      bulk:
+        index: test_counter
+        refresh: true
+        body:
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }'
+
+  - do:
+      search:
+        index: test_counter
+        body:
+          aggs:
+            counter_extended_stats:
+              stats:
+                field: counter
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - match: { aggregations.counter_extended_stats.count: 4 }
+  - close_to: { aggregations.counter_extended_stats.min: { value: 10.00, error: 0.01 } }
+  - close_to: { aggregations.counter_extended_stats.max: { value: 28.00, error: 0.01 } }
+  - close_to: { aggregations.counter_extended_stats.avg: { value: 20.00, error: 0.01 } }
+  - close_to: { aggregations.counter_extended_stats.sum: { value: 80.00, error: 0.01 } }
+
+---
+"sum":
+  - do:
+      bulk:
+        index: test_counter
+        refresh: true
+        body:
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }'
+
+  - do:
+      search:
+        index: test_counter
+        body:
+          aggs:
+            counter_sum:
+              sum:
+                field: counter
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - close_to: { aggregations.counter_sum.value: { value: 80.00, error: 0.01 } }
+
+---
+"value count":
+  - do:
+      bulk:
+        index: test_counter
+        refresh: true
+        body:
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:01:00Z", "key": "bar", "counter": 10 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:02:00Z", "key": "bar", "counter": 20 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:03:00Z", "key": "bar", "counter": 22 }'
+          - '{ "index": {} }'
+          - '{ "@timestamp": "2021-01-01T00:04:00Z", "key": "bar", "counter": 28 }'
+
+  - do:
+      search:
+        index: test_counter
+        body:
+          aggs:
+            counter_value_count:
+              value_count:
+                field: counter
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - match: { aggregations.counter_value_count.value: 4 }
+
+---
+"weighted avg":
+  - do:
+      search:
+        index: test_counter
+        body:
+          aggs:
+            counter_weighted_avg:
+              weighted_avg:
+                value:
+                  field: counter
+                weight:
+                  field: weight
+
+  - match: { hits.total.value: 4 }
+  - length: { hits.hits: 4 }
+  - close_to: { aggregations.counter_weighted_avg.value: { value: 18.66, error: 0.01 } }

server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.search.aggregations.CardinalityUpperBound;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
@@ -41,7 +42,12 @@ class AvgAggregatorFactory extends ValuesSourceAggregatorFactory {
     static void registerAggregators(ValuesSourceRegistry.Builder builder) {
         builder.register(
             AvgAggregationBuilder.REGISTRY_KEY,
-            List.of(CoreValuesSourceType.NUMERIC, CoreValuesSourceType.DATE, CoreValuesSourceType.BOOLEAN),
+            List.of(
+                CoreValuesSourceType.NUMERIC,
+                CoreValuesSourceType.DATE,
+                CoreValuesSourceType.BOOLEAN,
+                TimeSeriesValuesSourceType.COUNTER
+            ),
             AvgAggregator::new,
             true
         );