Skip to content

Commit bec10d7

Browse files
gmarouligmarouli
authored
Downsample exponential histograms (elastic#138561)
1 parent 4c33d20 commit bec10d7

File tree

21 files changed

+487
-72
lines changed

21 files changed

+487
-72
lines changed

‎libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@
3636
* while keeping the bucket count in the result below a given limit.
3737
*/
3838
public class ExponentialHistogramMerger implements Accountable, Releasable {
39-
39+
// OpenTelemetry SDK default, we might make this configurable later
40+
private static final int MAX_HISTOGRAM_BUCKETS = 320;
4041
private static final long BASE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ExponentialHistogramMerger.class) + DownscaleStats.SIZE;
4142

4243
// Our algorithm is not in-place, therefore we use two histograms and ping-pong between them
@@ -53,6 +54,14 @@ public class ExponentialHistogramMerger implements Accountable, Releasable {
5354
private final ExponentialHistogramCircuitBreaker circuitBreaker;
5455
private boolean closed = false;
5556

57+
/**
58+
* Creates a new instance with the OpenTelemetry SDK default bucket limit of {@link ExponentialHistogramMerger#MAX_HISTOGRAM_BUCKETS}
59+
* @param circuitBreaker the circuit breaker to use to limit memory allocations
60+
*/
61+
public static ExponentialHistogramMerger create(ExponentialHistogramCircuitBreaker circuitBreaker) {
62+
return create(MAX_HISTOGRAM_BUCKETS, circuitBreaker);
63+
}
64+
5665
/**
5766
* Creates a new instance with the specified bucket limit.
5867
*

‎x-pack/plugin/downsample/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ dependencies {
1717
testImplementation project(xpackModule('ilm'))
1818
compileOnly project(xpackModule('analytics'))
1919
compileOnly project(xpackModule('mapper-aggregate-metric'))
20+
compileOnly project(xpackModule('mapper-exponential-histogram'))
2021
testImplementation(testArtifact(project(xpackModule('core'))))
2122
testImplementation project(xpackModule('ccr'))
2223
testImplementation project(xpackModule('esql'))

‎x-pack/plugin/downsample/qa/rest/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ dependencies {
2121
clusterModules project(':modules:data-streams')
2222
clusterModules project(':modules:aggregations')
2323
clusterModules project(':modules:ingest-common')
24+
clusterModules project(xpackModule('mapper-exponential-histogram'))
2425
}
2526

2627
restResources {

‎x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,11 @@ public class DownsampleRestIT extends ESClientYamlSuiteTestCase {
2121
.module("x-pack-downsample")
2222
.module("x-pack-ilm")
2323
.module("lang-painless")
24-
.module("aggregations") // for auto_date_histogram
25-
.module("mapper-extras") // for scaled_float
26-
.module("x-pack-analytics") // for histogram
27-
.module("data-streams") // for time series
24+
.module("aggregations") // for auto_date_histogram
25+
.module("mapper-extras") // for scaled_float
26+
.module("x-pack-analytics") // for histogram
27+
.module("data-streams") // for time series
28+
.module("exponential-histogram")// for exponential histograms
2829
.module("ingest-common")
2930
.setting("xpack.license.self_generated.type", "trial")
3031
.setting("xpack.security.enabled", "false")

‎x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithBasicRestIT.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,12 @@ public class DownsampleWithBasicRestIT extends ESClientYamlSuiteTestCase {
2121
.module("x-pack-downsample")
2222
.module("x-pack-ilm")
2323
.module("lang-painless")
24-
.module("aggregations") // for auto_date_histogram
25-
.module("mapper-extras") // for scaled_float
26-
.module("x-pack-analytics") // for histogram
27-
.module("data-streams") // for time series
24+
.module("aggregations") // for auto_date_histogram
25+
.module("mapper-extras") // for scaled_float
26+
.module("x-pack-analytics") // for histogram
27+
.module("data-streams") // for time series
2828
.module("ingest-common")
29+
.module("exponential-histogram")// for exponential histograms
2930
.setting("xpack.security.enabled", "false")
3031
.build();
3132

‎x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithSecurityRestIT.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ public class DownsampleWithSecurityRestIT extends ESClientYamlSuiteTestCase {
2525
@ClassRule
2626
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
2727
.module("x-pack-downsample")
28-
.module("x-pack-analytics") // for histogram
28+
.module("x-pack-analytics") // for histogram
29+
.module("exponential-histogram")// for exponential histograms
2930
.setting("xpack.license.self_generated.type", "trial")
3031
.setting("xpack.security.enabled", "true")
3132
.user(USERNAME, PASSWORD)

‎x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml‎

Lines changed: 170 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ setup:
346346
- match: { hits.hits.0._source.k8s\.pod\.values: [1, 1, 2] }
347347
- is_false: hits.hits.0._source.k8s\.pod\.running
348348

349-
# Assert rollup index settings
349+
# Assert downsample index settings
350350
- do:
351351
indices.get_settings:
352352
index: test-downsample
@@ -358,7 +358,7 @@ setup:
358358
- match: { test-downsample.settings.index.downsample.source.name: test }
359359
- match: { test-downsample.settings.index.number_of_shards: "1" }
360360

361-
# Assert rollup index mapping
361+
# Assert downsample index mapping
362362
- do:
363363
indices.get_mapping:
364364
index: test-downsample
@@ -432,7 +432,7 @@ setup:
432432
- match: { hits.hits.0._source.k8s\.pod\.values: [1, 1, 2] }
433433
- is_false: hits.hits.0._source.k8s\.pod\.running
434434

435-
# Assert rollup index settings
435+
# Assert downsample index settings
436436
- do:
437437
indices.get_settings:
438438
index: test-downsample
@@ -442,7 +442,7 @@ setup:
442442
- match: { test-downsample.settings.index.time_series.start_time: 2021-04-28T00:00:00Z }
443443
- match: { test-downsample.settings.index.routing_path: [ "metricset", "k8s.pod.uid"] }
444444
- match: { test-downsample.settings.index.downsample.source.name: test }
445-
- match: { test-downsample.settings.index.downsample.method: last_value }
445+
- match: { test-downsample.settings.index.downsample.sampling_method: last_value }
446446
- match: { test-downsample.settings.index.number_of_shards: "1" }
447447

448448
# Assert rollup index mapping
@@ -1001,7 +1001,7 @@ setup:
10011001
- do:
10021002
catch: /Downsampling method \[aggregate\] is not compatible with the source index downsampling method \[last_value\]/
10031003
indices.downsample:
1004-
index: test-downsample
1004+
index: test-downsample-last-value
10051005
target_index: test-downsample-2
10061006
body: >
10071007
{
@@ -1846,3 +1846,168 @@ setup:
18461846
- match: { hits.hits.1._source._doc_count: 1 }
18471847
- match: { hits.hits.1._source.k8s\.pod\.name: cat }
18481848
- match: { hits.hits.1._source.k8s\.pod\.empty: "" }
1849+
1850+
---
1851+
"Downsample exponential histogram with last value":
1852+
- requires:
1853+
capabilities:
1854+
- method: POST
1855+
path: /{index}/_downsample/{target_index}
1856+
capabilities: [ "downsample.sampling_mode.last_value", "downsampling.exponential_histograms" ]
1857+
test_runner_features: [ "capabilities" ]
1858+
reason: Last value sampling method was added in 9.3
1859+
- do:
1860+
indices.create:
1861+
index: test-exponential-histogram
1862+
body:
1863+
settings:
1864+
number_of_shards: 1
1865+
index:
1866+
mode: time_series
1867+
routing_path: [ metricset ]
1868+
time_series:
1869+
start_time: 2021-04-28T00:00:00Z
1870+
end_time: 2021-04-29T00:00:00Z
1871+
mappings:
1872+
properties:
1873+
"@timestamp":
1874+
type: date
1875+
metricset:
1876+
type: keyword
1877+
time_series_dimension: true
1878+
metric:
1879+
type: exponential_histogram
1880+
time_series_metric: histogram
1881+
label:
1882+
type: exponential_histogram
1883+
- is_true: shards_acknowledged
1884+
1885+
- do:
1886+
bulk:
1887+
refresh: true
1888+
index: test-exponential-histogram
1889+
body:
1890+
- '{"index": {}}'
1891+
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "lala", "metric": {"scale": -1, "sum": 1.2295711150758473, "min": -0.9714203537912545, "max": 0.9865998839317596, "negative": {"indices": [-3, -2, -1], "counts": [2, 11, 36]}, "positive": {"indices": [-6, -3, -2, -1], "counts": [1, 3, 11, 36]}}, "label": {"scale": -1, "sum": 1.2295711150758473, "min": -0.9714203537912545, "max": 0.9865998839317596, "negative": {"indices": [-3, -2, -1], "counts": [2, 11, 36]}, "positive": {"indices": [-6, -3, -2, -1], "counts": [1, 3, 11, 36]}}}'
1892+
- '{"index": {}}'
1893+
- '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "lala", "metric": {"scale": 1, "sum": 5.89290987609197, "min": -0.9410989907817573, "max": 0.9821606962370899, "negative": {"indices": [-9, -8, -7, -4, -3, -2, -1], "counts": [1, 2, 1, 3, 3, 2, 6]}, "positive": {"indices": [-10, -7, -6, -5, -4, -3, -2, -1], "counts": [3, 2, 1, 2, 5, 6, 4, 9]}}, "label": {"scale": 1, "sum": 5.89290987609197, "min": -0.9410989907817573, "max": 0.9821606962370899, "negative": {"indices": [-9, -8, -7, -4, -3, -2, -1], "counts": [1, 2, 1, 3, 3, 2, 6]}, "positive": {"indices": [-10, -7, -6, -5, -4, -3, -2, -1], "counts": [3, 2, 1, 2, 5, 6, 4, 9]}}}'
1894+
- match: { items.0.index.result: "created"}
1895+
- match: { items.1.index.result: "created"}
1896+
1897+
- do:
1898+
indices.put_settings:
1899+
index: test-exponential-histogram
1900+
body:
1901+
index.blocks.write: true
1902+
- is_true: acknowledged
1903+
1904+
- do:
1905+
indices.downsample:
1906+
index: test-exponential-histogram
1907+
target_index: downsampled-exponential-histogram
1908+
body: >
1909+
{
1910+
"fixed_interval": "1h",
1911+
"sampling_method": "last_value"
1912+
}
1913+
- is_true: acknowledged
1914+
1915+
- do:
1916+
search:
1917+
index: downsampled-exponential-histogram
1918+
1919+
- length: { hits.hits: 1 }
1920+
1921+
- match: { hits.hits.0._source._doc_count: 2 }
1922+
- match: { hits.hits.0._source.metricset: lala }
1923+
- match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" }
1924+
- match: { hits.hits.0._source.metric.max: 0.9821606962370899 }
1925+
- match: { hits.hits.0._source.metric.min: -0.9410989907817573 }
1926+
- match: { hits.hits.0._source.metric.scale: 1 }
1927+
- match: { hits.hits.0._source.metric.sum: 5.89290987609197 }
1928+
- match: { hits.hits.0._source.label.max: 0.9821606962370899 }
1929+
- match: { hits.hits.0._source.label.min: -0.9410989907817573 }
1930+
- match: { hits.hits.0._source.label.scale: 1 }
1931+
- match: { hits.hits.0._source.label.sum: 5.89290987609197 }
1932+
1933+
---
1934+
"Downsample exponential histogram with aggregate":
1935+
- requires:
1936+
capabilities:
1937+
- method: POST
1938+
path: /{index}/_downsample/{target_index}
1939+
capabilities: [ "downsampling.exponential_histograms" ]
1940+
test_runner_features: [ "capabilities" ]
1941+
reason: Last value sampling method was added in 9.3
1942+
- do:
1943+
indices.create:
1944+
index: test-exponential-histogram
1945+
body:
1946+
settings:
1947+
number_of_shards: 1
1948+
index:
1949+
mode: time_series
1950+
routing_path: [ metricset ]
1951+
time_series:
1952+
start_time: 2021-04-28T00:00:00Z
1953+
end_time: 2021-04-29T00:00:00Z
1954+
mappings:
1955+
properties:
1956+
"@timestamp":
1957+
type: date
1958+
metricset:
1959+
type: keyword
1960+
time_series_dimension: true
1961+
metric:
1962+
type: exponential_histogram
1963+
time_series_metric: histogram
1964+
label:
1965+
type: exponential_histogram
1966+
- is_true: shards_acknowledged
1967+
1968+
- do:
1969+
bulk:
1970+
refresh: true
1971+
index: test-exponential-histogram
1972+
body:
1973+
- '{"index": {}}'
1974+
- '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "lala", "metric": {"scale":38,"sum":-2.659439830689209,"min":-0.9985699620066124,"max":0.9991606633362218,"positive":{"indices":[-1349015837690,-1133741949749,-1093555204471,-878875428260,-870694284095,-831557211207,-801793317707,-633655864830,-583473541038,-576512037819,-568470393517,-555322469609,-510705457264,-497886869173,-485920366989,-459535642468,-433595368145,-390027945362,-356113581539,-343989550188,-340618512768,-292750567759,-278017998597,-273632142061,-265328573027,-262313863416,-261799809534,-243544433917,-200039973732,-189492605429,-153154647965,-134459061772,-117996065668,-111711978805,-109965984611,-101785856856,-93187164407,-79681306932,-76611651392,-65149069301,-58423697696,-56765280861,-41035363279,-39729265577,-7007278892,-332991304],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1722041245251,-1507876068779,-1461993774485,-1437177146905,-1022221748846,-909329959788,-902070008615,-869527704039,-859349507080,-836401169293,-835541219174,-827446708753,-823853818576,-767042645194,-764466131268,-698239247988,-598695656860,-560446067886,-554301612745,-521812328033,-508700015492,-462218888037,-417010270052,-384781248780,-371405487026,-366789447120,-349996665670,-300317310192,-287581746174,-287132961026,-284865058959,-255687331935,-236975923503,-200065874687,-196100184062,-147243768507,-145933242978,-140904484007,-136697299301,-117022220796,-87401418251,-63863967756,-62902280835,-59450473434,-53232718483,-49452610935,-45384224466,-36361788727,-34176300034,-12469429575,-9431189993,-3254481828,-2083495024,-567508884],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}, "label": {"scale":38,"sum":-2.659439830689209,"min":-0.9985699620066124,"max":0.9991606633362218,"positive":{"indices":[-1349015837690,-1133741949749,-1093555204471,-878875428260,-870694284095,-831557211207,-801793317707,-633655864830,-583473541038,-576512037819,-568470393517,-555322469609,-510705457264,-497886869173,-485920366989,-459535642468,-433595368145,-390027945362,-356113581539,-343989550188,-340618512768,-292750567759,-278017998597,-273632142061,-265328573027,-262313863416,-261799809534,-243544433917,-200039973732,-189492605429,-153154647965,-134459061772,-117996065668,-111711978805,-109965984611,-101785856856,-93187164407,-79681306932,-76611651392,-65149069301,-58423697696,-56765280861,-41035363279,-39729265577,-7007278892,-332991304],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1722041245251,-1507876068779,-1461993774485,-1437177146905,-1022221748846,-909329959788,-902070008615,-869527704039,-859349507080,-836401169293,-835541219174,-827446708753,-823853818576,-767042645194,-764466131268,-698239247988,-598695656860,-560446067886,-554301612745,-521812328033,-508700015492,-462218888037,-417010270052,-384781248780,-371405487026,-366789447120,-349996665670,-300317310192,-287581746174,-287132961026,-284865058959,-255687331935,-236975923503,-200065874687,-196100184062,-147243768507,-145933242978,-140904484007,-136697299301,-117022220796,-87401418251,-63863967756,-62902280835,-59450473434,-53232718483,-49452610935,-45384224466,-36361788727,-34176300034,-12469429575,-9431189993,-3254481828,-2083495024,-567508884],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}}'
1975+
- '{"index": {}}'
1976+
- '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "lala", "metric": {"scale":38,"sum":-2.612341035444145,"min":-0.8596842630789088,"max":0.9412120998532649,"positive":{"indices":[-1527782821719,-1491647186305,-912103649399,-744031889051,-669036239523,-602111557612,-586601121715,-567461548128,-483021101785,-450865160617,-331097834736,-293981770197,-289290353357,-246965149109,-203070759894,-180702903638,-135653049938,-93843302593,-78356307212,-76729181967,-65070806329,-64241350555,-24026590621],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1423777207002,-923870284341,-877992064499,-870206177266,-850685295282,-759799372292,-643924938249,-570457241089,-528116212727,-369003212885,-315079526687,-292745353109,-235218433328,-222024648135,-204603508425,-195434414609,-177993661419,-172367743205,-168351131368,-149696012047,-115115447928,-95206374327,-74115574931,-74013160315,-64191774880,-63955626713,-59956698242],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}, "label": {"scale":38,"sum":-2.612341035444145,"min":-0.8596842630789088,"max":0.9412120998532649,"positive":{"indices":[-1527782821719,-1491647186305,-912103649399,-744031889051,-669036239523,-602111557612,-586601121715,-567461548128,-483021101785,-450865160617,-331097834736,-293981770197,-289290353357,-246965149109,-203070759894,-180702903638,-135653049938,-93843302593,-78356307212,-76729181967,-65070806329,-64241350555,-24026590621],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1423777207002,-923870284341,-877992064499,-870206177266,-850685295282,-759799372292,-643924938249,-570457241089,-528116212727,-369003212885,-315079526687,-292745353109,-235218433328,-222024648135,-204603508425,-195434414609,-177993661419,-172367743205,-168351131368,-149696012047,-115115447928,-95206374327,-74115574931,-74013160315,-64191774880,-63955626713,-59956698242],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}}'
1977+
- match: { items.0.index.result: "created"}
1978+
- match: { items.1.index.result: "created"}
1979+
1980+
- do:
1981+
indices.put_settings:
1982+
index: test-exponential-histogram
1983+
body:
1984+
index.blocks.write: true
1985+
- is_true: acknowledged
1986+
1987+
- do:
1988+
indices.downsample:
1989+
index: test-exponential-histogram
1990+
target_index: downsampled-exponential-histogram-aggregate
1991+
body: >
1992+
{
1993+
"fixed_interval": "1h"
1994+
}
1995+
- is_true: acknowledged
1996+
1997+
- do:
1998+
search:
1999+
index: downsampled-exponential-histogram-aggregate
2000+
2001+
- length: { hits.hits: 1 }
2002+
2003+
- match: { hits.hits.0._source._doc_count: 2 }
2004+
- match: { hits.hits.0._source.metricset: lala }
2005+
- match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" }
2006+
- match: { hits.hits.0._source.metric.max: 0.9991606633362218 }
2007+
- match: { hits.hits.0._source.metric.min: -0.9985699620066124 }
2008+
- match: { hits.hits.0._source.metric.scale: 38 }
2009+
- match: { hits.hits.0._source.metric.sum: -5.271780866133354 }
2010+
- match: { hits.hits.0._source.label.max: 0.9412120998532649 }
2011+
- match: { hits.hits.0._source.label.min: -0.8596842630789088 }
2012+
- match: { hits.hits.0._source.label.scale: 38 }
2013+
- match: { hits.hits.0._source.label.sum: -2.612341035444145 }

0 commit comments

Comments
 (0)