Downsample exponential histograms

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java

@@ -36,7 +36,8 @@
  * while keeping the bucket count in the result below a given limit.
  */
 public class ExponentialHistogramMerger implements Accountable, Releasable {
-
+    // OpenTelemetry SDK default, we might make this configurable later
+    private static final int MAX_HISTOGRAM_BUCKETS = 320;
     private static final long BASE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ExponentialHistogramMerger.class) + DownscaleStats.SIZE;
 
     // Our algorithm is not in-place, therefore we use two histograms and ping-pong between them
@@ -53,6 +54,14 @@ public class ExponentialHistogramMerger implements Accountable, Releasable {
     private final ExponentialHistogramCircuitBreaker circuitBreaker;
     private boolean closed = false;
 
+    /**
+     * Creates a new instance with the OpenTelemetry SDK default bucket limit of {@link ExponentialHistogramMerger#MAX_HISTOGRAM_BUCKETS}
+     * @param circuitBreaker the circuit breaker to use to limit memory allocations
+     */
+    public static ExponentialHistogramMerger create(ExponentialHistogramCircuitBreaker circuitBreaker) {
+        return create(MAX_HISTOGRAM_BUCKETS, circuitBreaker);
+    }
+
     /**
      * Creates a new instance with the specified bucket limit.
      *

x-pack/plugin/downsample/build.gradle

@@ -17,6 +17,7 @@ dependencies {
   testImplementation project(xpackModule('ilm'))
   compileOnly project(xpackModule('analytics'))
   compileOnly project(xpackModule('mapper-aggregate-metric'))
+  compileOnly project(xpackModule('mapper-exponential-histogram'))
   testImplementation(testArtifact(project(xpackModule('core'))))
   testImplementation project(xpackModule('ccr'))
   testImplementation project(xpackModule('esql'))

x-pack/plugin/downsample/qa/rest/build.gradle

@@ -21,6 +21,7 @@ dependencies {
   clusterModules project(':modules:data-streams')
   clusterModules project(':modules:aggregations')
   clusterModules project(':modules:ingest-common')
+  clusterModules project(xpackModule('mapper-exponential-histogram'))
 }
 
 restResources {

x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java

@@ -21,10 +21,11 @@ public class DownsampleRestIT extends ESClientYamlSuiteTestCase {
         .module("x-pack-downsample")
         .module("x-pack-ilm")
         .module("lang-painless")
-        .module("aggregations")     // for auto_date_histogram
-        .module("mapper-extras")    // for scaled_float
-        .module("x-pack-analytics") // for histogram
-        .module("data-streams")     // for time series
+        .module("aggregations")         // for auto_date_histogram
+        .module("mapper-extras")        // for scaled_float
+        .module("x-pack-analytics")     // for histogram
+        .module("data-streams")         // for time series
+        .module("exponential-histogram")// for exponential histograms
         .module("ingest-common")
         .setting("xpack.license.self_generated.type", "trial")
         .setting("xpack.security.enabled", "false")

x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithBasicRestIT.java

@@ -21,11 +21,12 @@ public class DownsampleWithBasicRestIT extends ESClientYamlSuiteTestCase {
         .module("x-pack-downsample")
         .module("x-pack-ilm")
         .module("lang-painless")
-        .module("aggregations")     // for auto_date_histogram
-        .module("mapper-extras")    // for scaled_float
-        .module("x-pack-analytics") // for histogram
-        .module("data-streams")     // for time series
+        .module("aggregations")         // for auto_date_histogram
+        .module("mapper-extras")        // for scaled_float
+        .module("x-pack-analytics")     // for histogram
+        .module("data-streams")         // for time series
         .module("ingest-common")
+        .module("exponential-histogram")// for exponential histograms
         .setting("xpack.security.enabled", "false")
         .build();
 

x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithSecurityRestIT.java

@@ -25,7 +25,8 @@ public class DownsampleWithSecurityRestIT extends ESClientYamlSuiteTestCase {
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("x-pack-downsample")
-        .module("x-pack-analytics") // for histogram
+        .module("x-pack-analytics")     // for histogram
+        .module("exponential-histogram")// for exponential histograms
         .setting("xpack.license.self_generated.type", "trial")
         .setting("xpack.security.enabled", "true")
         .user(USERNAME, PASSWORD)

x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml

@@ -346,7 +346,7 @@ setup:
   - match:  { hits.hits.0._source.k8s\.pod\.values: [1, 1, 2] }
   - is_false: hits.hits.0._source.k8s\.pod\.running
 
-  # Assert rollup index settings
+  # Assert downsample index settings
   - do:
       indices.get_settings:
         index: test-downsample
@@ -358,7 +358,7 @@ setup:
   - match: { test-downsample.settings.index.downsample.source.name: test }
   - match: { test-downsample.settings.index.number_of_shards: "1" }
 
-  # Assert rollup index mapping
+  # Assert downsample index mapping
   - do:
       indices.get_mapping:
         index: test-downsample
@@ -432,7 +432,7 @@ setup:
   - match:  { hits.hits.0._source.k8s\.pod\.values: [1, 1, 2] }
   - is_false: hits.hits.0._source.k8s\.pod\.running
 
-  # Assert rollup index settings
+  # Assert downsample index settings
   - do:
       indices.get_settings:
         index: test-downsample
@@ -442,7 +442,7 @@ setup:
   - match: { test-downsample.settings.index.time_series.start_time: 2021-04-28T00:00:00Z }
   - match: { test-downsample.settings.index.routing_path: [ "metricset", "k8s.pod.uid"] }
   - match: { test-downsample.settings.index.downsample.source.name: test }
-  - match: { test-downsample.settings.index.downsample.method: last_value }
+  - match: { test-downsample.settings.index.downsample.sampling_method: last_value }
   - match: { test-downsample.settings.index.number_of_shards: "1" }
 
   # Assert rollup index mapping
@@ -1001,7 +1001,7 @@ setup:
   - do:
       catch: /Downsampling method \[aggregate\] is not compatible with the source index downsampling method \[last_value\]/
       indices.downsample:
-        index: test-downsample
+        index: test-downsample-last-value
         target_index: test-downsample-2
         body: >
           {
@@ -1846,3 +1846,168 @@ setup:
   - match: { hits.hits.1._source._doc_count: 1 }
   - match: { hits.hits.1._source.k8s\.pod\.name: cat }
   - match: { hits.hits.1._source.k8s\.pod\.empty: "" }
+
+---
+"Downsample exponential histogram with last value":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /{index}/_downsample/{target_index}
+          capabilities: [ "downsample.sampling_mode.last_value", "downsampling.exponential_histograms" ]
+      test_runner_features: [ "capabilities" ]
+      reason: Last value sampling method was added in 9.3
+  - do:
+      indices.create:
+        index: test-exponential-histogram
+        body:
+          settings:
+            number_of_shards: 1
+            index:
+              mode: time_series
+              routing_path: [ metricset ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+              metric:
+                type: exponential_histogram
+                time_series_metric: histogram
+              label:
+                type: exponential_histogram
+  - is_true: shards_acknowledged
+
+  - do:
+      bulk:
+        refresh: true
+        index: test-exponential-histogram
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "lala", "metric": {"scale": -1, "sum": 1.2295711150758473, "min": -0.9714203537912545, "max": 0.9865998839317596, "negative": {"indices": [-3, -2, -1], "counts": [2, 11, 36]}, "positive": {"indices": [-6, -3, -2, -1], "counts": [1, 3, 11, 36]}}, "label": {"scale": -1, "sum": 1.2295711150758473, "min": -0.9714203537912545, "max": 0.9865998839317596, "negative": {"indices": [-3, -2, -1], "counts": [2, 11, 36]}, "positive": {"indices": [-6, -3, -2, -1], "counts": [1, 3, 11, 36]}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "lala", "metric": {"scale": 1, "sum": 5.89290987609197, "min": -0.9410989907817573, "max": 0.9821606962370899, "negative": {"indices": [-9, -8, -7, -4, -3, -2, -1], "counts": [1, 2, 1, 3, 3, 2, 6]}, "positive": {"indices": [-10, -7, -6, -5, -4, -3, -2, -1], "counts": [3, 2, 1, 2, 5, 6, 4, 9]}}, "label": {"scale": 1, "sum": 5.89290987609197, "min": -0.9410989907817573, "max": 0.9821606962370899, "negative": {"indices": [-9, -8, -7, -4, -3, -2, -1], "counts": [1, 2, 1, 3, 3, 2, 6]}, "positive": {"indices": [-10, -7, -6, -5, -4, -3, -2, -1], "counts": [3, 2, 1, 2, 5, 6, 4, 9]}}}'
+  - match: { items.0.index.result: "created"}
+  - match: { items.1.index.result: "created"}
+
+  - do:
+      indices.put_settings:
+        index: test-exponential-histogram
+        body:
+          index.blocks.write: true
+  - is_true: acknowledged
+
+  - do:
+      indices.downsample:
+        index: test-exponential-histogram
+        target_index: downsampled-exponential-histogram
+        body: >
+          {
+            "fixed_interval": "1h",
+            "sampling_method": "last_value"
+          }
+  - is_true: acknowledged
+
+  - do:
+      search:
+        index: downsampled-exponential-histogram
+
+  - length: { hits.hits: 1 }
+
+  - match: { hits.hits.0._source._doc_count: 2 }
+  - match: { hits.hits.0._source.metricset: lala }
+  - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" }
+  - match: { hits.hits.0._source.metric.max: 0.9821606962370899 }
+  - match: { hits.hits.0._source.metric.min: -0.9410989907817573 }
+  - match: { hits.hits.0._source.metric.scale: 1 }
+  - match: { hits.hits.0._source.metric.sum: 5.89290987609197 }
+  - match: { hits.hits.0._source.label.max: 0.9821606962370899 }
+  - match: { hits.hits.0._source.label.min: -0.9410989907817573 }
+  - match: { hits.hits.0._source.label.scale: 1 }
+  - match: { hits.hits.0._source.label.sum: 5.89290987609197 }
+
+---
+"Downsample exponential histogram with aggregate":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /{index}/_downsample/{target_index}
+          capabilities: [ "downsampling.exponential_histograms" ]
+      test_runner_features: [ "capabilities" ]
+      reason: Last value sampling method was added in 9.3
+  - do:
+      indices.create:
+        index: test-exponential-histogram
+        body:
+          settings:
+            number_of_shards: 1
+            index:
+              mode: time_series
+              routing_path: [ metricset ]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+              metric:
+                type: exponential_histogram
+                time_series_metric: histogram
+              label:
+                type: exponential_histogram
+  - is_true: shards_acknowledged
+
+  - do:
+      bulk:
+        refresh: true
+        index: test-exponential-histogram
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "lala", "metric": {"scale":38,"sum":-2.659439830689209,"min":-0.9985699620066124,"max":0.9991606633362218,"positive":{"indices":[-1349015837690,-1133741949749,-1093555204471,-878875428260,-870694284095,-831557211207,-801793317707,-633655864830,-583473541038,-576512037819,-568470393517,-555322469609,-510705457264,-497886869173,-485920366989,-459535642468,-433595368145,-390027945362,-356113581539,-343989550188,-340618512768,-292750567759,-278017998597,-273632142061,-265328573027,-262313863416,-261799809534,-243544433917,-200039973732,-189492605429,-153154647965,-134459061772,-117996065668,-111711978805,-109965984611,-101785856856,-93187164407,-79681306932,-76611651392,-65149069301,-58423697696,-56765280861,-41035363279,-39729265577,-7007278892,-332991304],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1722041245251,-1507876068779,-1461993774485,-1437177146905,-1022221748846,-909329959788,-902070008615,-869527704039,-859349507080,-836401169293,-835541219174,-827446708753,-823853818576,-767042645194,-764466131268,-698239247988,-598695656860,-560446067886,-554301612745,-521812328033,-508700015492,-462218888037,-417010270052,-384781248780,-371405487026,-366789447120,-349996665670,-300317310192,-287581746174,-287132961026,-284865058959,-255687331935,-236975923503,-200065874687,-196100184062,-147243768507,-145933242978,-140904484007,-136697299301,-117022220796,-87401418251,-63863967756,-62902280835,-59450473434,-53232718483,-49452610935,-45384224466,-36361788727,-34176300034,-12469429575,-9431189993,-3254481828,-2083495024,-567508884],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}, "label": {"scale":38,"sum":-2.659439830689209,"min":-0.9985699620066124,"max":0.9991606633362218,"positive":{"indices":[-1349015837690,-1133741949749,-1093555204471,-878875428260,-870694284095,-831557211207,-801793317707,-633655864830,-583473541038,-576512037819,-568470393517,-555322469609,-510705457264,-497886869173,-485920366989,-459535642468,-433595368145,-390027945362,-356113581539,-343989550188,-340618512768,-292750567759,-278017998597,-273632142061,-265328573027,-262313863416,-261799809534,-243544433917,-200039973732,-189492605429,-153154647965,-134459061772,-117996065668,-111711978805,-109965984611,-101785856856,-93187164407,-79681306932,-76611651392,-65149069301,-58423697696,-56765280861,-41035363279,-39729265577,-7007278892,-332991304],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1722041245251,-1507876068779,-1461993774485,-1437177146905,-1022221748846,-909329959788,-902070008615,-869527704039,-859349507080,-836401169293,-835541219174,-827446708753,-823853818576,-767042645194,-764466131268,-698239247988,-598695656860,-560446067886,-554301612745,-521812328033,-508700015492,-462218888037,-417010270052,-384781248780,-371405487026,-366789447120,-349996665670,-300317310192,-287581746174,-287132961026,-284865058959,-255687331935,-236975923503,-200065874687,-196100184062,-147243768507,-145933242978,-140904484007,-136697299301,-117022220796,-87401418251,-63863967756,-62902280835,-59450473434,-53232718483,-49452610935,-45384224466,-36361788727,-34176300034,-12469429575,-9431189993,-3254481828,-2083495024,-567508884],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "lala", "metric": {"scale":38,"sum":-2.612341035444145,"min":-0.8596842630789088,"max":0.9412120998532649,"positive":{"indices":[-1527782821719,-1491647186305,-912103649399,-744031889051,-669036239523,-602111557612,-586601121715,-567461548128,-483021101785,-450865160617,-331097834736,-293981770197,-289290353357,-246965149109,-203070759894,-180702903638,-135653049938,-93843302593,-78356307212,-76729181967,-65070806329,-64241350555,-24026590621],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1423777207002,-923870284341,-877992064499,-870206177266,-850685295282,-759799372292,-643924938249,-570457241089,-528116212727,-369003212885,-315079526687,-292745353109,-235218433328,-222024648135,-204603508425,-195434414609,-177993661419,-172367743205,-168351131368,-149696012047,-115115447928,-95206374327,-74115574931,-74013160315,-64191774880,-63955626713,-59956698242],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}, "label": {"scale":38,"sum":-2.612341035444145,"min":-0.8596842630789088,"max":0.9412120998532649,"positive":{"indices":[-1527782821719,-1491647186305,-912103649399,-744031889051,-669036239523,-602111557612,-586601121715,-567461548128,-483021101785,-450865160617,-331097834736,-293981770197,-289290353357,-246965149109,-203070759894,-180702903638,-135653049938,-93843302593,-78356307212,-76729181967,-65070806329,-64241350555,-24026590621],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1423777207002,-923870284341,-877992064499,-870206177266,-850685295282,-759799372292,-643924938249,-570457241089,-528116212727,-369003212885,-315079526687,-292745353109,-235218433328,-222024648135,-204603508425,-195434414609,-177993661419,-172367743205,-168351131368,-149696012047,-115115447928,-95206374327,-74115574931,-74013160315,-64191774880,-63955626713,-59956698242],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}}'
+  - match: { items.0.index.result: "created"}
+  - match: { items.1.index.result: "created"}
+
+  - do:
+      indices.put_settings:
+        index: test-exponential-histogram
+        body:
+          index.blocks.write: true
+  - is_true: acknowledged
+
+  - do:
+      indices.downsample:
+        index: test-exponential-histogram
+        target_index: downsampled-exponential-histogram-aggregate
+        body: >
+          {
+            "fixed_interval": "1h"
+          }
+  - is_true: acknowledged
+
+  - do:
+      search:
+        index: downsampled-exponential-histogram-aggregate
+
+  - length: { hits.hits: 1 }
+
+  - match: { hits.hits.0._source._doc_count: 2 }
+  - match: { hits.hits.0._source.metricset: lala }
+  - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" }
+  - match: { hits.hits.0._source.metric.max: 0.9991606633362218 }
+  - match: { hits.hits.0._source.metric.min: -0.9985699620066124 }
+  - match: { hits.hits.0._source.metric.scale: 38 }
+  - match: { hits.hits.0._source.metric.sum: -5.271780866133354 }
+  - match: { hits.hits.0._source.label.max: 0.9412120998532649 }
+  - match: { hits.hits.0._source.label.min: -0.8596842630789088 }
+  - match: { hits.hits.0._source.label.scale: 38 }
+  - match: { hits.hits.0._source.label.sum: -2.612341035444145 }