OOM on expensive composite agg #136165
Description
Expensive composite aggs can lead to OOM during reduction phase.
The circuit breaker is supposed to catch these situations, but apparently it doesn't cover all of them.
There was a first attempt to fix it, but apparently the problem is still there (stack trace is from 9.1.5)
org.elasticsearch.common.util.concurrent.EsExecutors$EsThread @ 0x103a075ae8
at java.lang.OutOfMemoryError.<init>()V (OutOfMemoryError.java:48)
at org.elasticsearch.common.io.stream.StreamInput.readBytesReference(I)Lorg/elasticsearch/common/bytes/BytesReference; (StreamInput.java:196)
at org.elasticsearch.common.io.stream.StreamInput.readBytesReference()Lorg/elasticsearch/common/bytes/BytesReference; (StreamInput.java:119)
at org.elasticsearch.search.SearchHit.readFrom(Lorg/elasticsearch/common/io/stream/StreamInput;Z)Lorg/elasticsearch/search/SearchHit; (SearchHit.java:211)
at org.elasticsearch.search.SearchHits.readFrom(Lorg/elasticsearch/common/io/stream/StreamInput;Z)Lorg/elasticsearch/search/SearchHits; (SearchHits.java:138)
at org.elasticsearch.search.aggregations.metrics.InternalTopHits.<init>(Lorg/elasticsearch/common/io/stream/StreamInput;)V (InternalTopHits.java:69)
at org.elasticsearch.search.SearchModule$$Lambda+0x000000008f971158.read(Lorg/elasticsearch/common/io/stream/StreamInput;)Ljava/lang/Object; (Unknown Source)
at org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput.readNamedWriteableCollectionAsList(Ljava/lang/Class;)Ljava/util/List; (NamedWriteableAwareStreamInput.java:46)
at org.elasticsearch.search.aggregations.InternalAggregations.readFrom(Lorg/elasticsearch/common/io/stream/StreamInput;)Lorg/elasticsearch/search/aggregations/InternalAggregations; (InternalAggregations.java:146)
at org.elasticsearch.search.aggregations.bucket.composite.InternalComposite$InternalBucket.<init>(Lorg/elasticsearch/common/io/stream/StreamInput;Ljava/util/List;Ljava/util/List;)V (InternalComposite.java:354)
at org.elasticsearch.search.aggregations.bucket.composite.InternalComposite.lambda$new$1(Lorg/elasticsearch/common/io/stream/StreamInput;)Lorg/elasticsearch/search/aggregations/bucket/composite/InternalComposite$InternalBucket; (InternalComposite.java:105)
at org.elasticsearch.search.aggregations.bucket.composite.InternalComposite$$Lambda+0x00000000911f2d38.read(Lorg/elasticsearch/common/io/stream/StreamInput;)Ljava/lang/Object; (Unknown Source)
at org.elasticsearch.common.io.stream.StreamInput.readCollection(Lorg/elasticsearch/common/io/stream/Writeable$Reader;Ljava/util/function/IntFunction;Ljava/util/Collection;)Ljava/util/Collection; (StreamInput.java:1338)
at org.elasticsearch.common.io.stream.StreamInput.readCollectionAsList(Lorg/elasticsearch/common/io/stream/Writeable$Reader;)Ljava/util/List; (StreamInput.java:1210)
at org.elasticsearch.search.aggregations.bucket.composite.InternalComposite.<init>(Lorg/elasticsearch/common/io/stream/StreamInput;)V (InternalComposite.java:105)
at org.elasticsearch.search.SearchModule$$Lambda+0x000000008f979cd8.read(Lorg/elasticsearch/common/io/stream/StreamInput;)Ljava/lang/Object; (Unknown Source)
at org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput.readNamedWriteableCollectionAsList(Ljava/lang/Class;)Ljava/util/List; (NamedWriteableAwareStreamInput.java:46)
at org.elasticsearch.search.aggregations.InternalAggregations.readFrom(Lorg/elasticsearch/common/io/stream/StreamInput;)Lorg/elasticsearch/search/aggregations/InternalAggregations; (InternalAggregations.java:146)
at org.elasticsearch.search.query.QuerySearchResult$$Lambda+0x00000000911f26c0.read(Lorg/elasticsearch/common/io/stream/StreamInput;)Ljava/lang/Object; (Unknown Source)
at org.elasticsearch.common.io.stream.DelayableWriteable.deserialize(Lorg/elasticsearch/common/io/stream/Writeable$Reader;Lorg/elasticsearch/TransportVersion;Lorg/elasticsearch/common/io/stream/NamedWriteableRegistry;Lorg/elasticsearch/common/bytes/BytesReference;)Ljava/lang/Object; (DelayableWriteable.java:256)
at org.elasticsearch.common.io.stream.DelayableWriteable$Serialized.expand()Lorg/elasticsearch/common/io/stream/Writeable; (DelayableWriteable.java:203)
at org.elasticsearch.action.search.QueryPhaseResultConsumer.lambda$aggregate$1(Lorg/elasticsearch/search/query/QuerySearchResult;)Lorg/elasticsearch/search/aggregations/InternalAggregations; (QueryPhaseResultConsumer.java:419)
at org.elasticsearch.action.search.QueryPhaseResultConsumer$$Lambda+0x0000000091032a50.apply(Ljava/lang/Object;)Ljava/lang/Object; (Unknown Source)
at org.elasticsearch.common.collect.Iterators$MapIterator.lambda$forEachRemaining$0(Ljava/util/function/Consumer;Ljava/lang/Object;)V (Iterators.java:187)
at org.elasticsearch.common.collect.Iterators$MapIterator$$Lambda+0x00000000912d1900.accept(Ljava/lang/Object;)V (Unknown Source)
at java.util.ArrayList$Itr.forEachRemaining(Ljava/util/function/Consumer;)V (ArrayList.java:1086)
at org.elasticsearch.common.collect.Iterators$MapIterator.forEachRemaining(Ljava/util/function/Consumer;)V (Iterators.java:187)
at org.elasticsearch.search.aggregations.InternalAggregations.reduce(Ljava/util/Iterator;ILorg/elasticsearch/search/aggregations/AggregationReduceContext;)Lorg/elasticsearch/search/aggregations/InternalAggregations; (InternalAggregations.java:250)
at org.elasticsearch.search.aggregations.InternalAggregations.topLevelReduce(Ljava/util/Iterator;ILorg/elasticsearch/search/aggregations/AggregationReduceContext;)Lorg/elasticsearch/search/aggregations/InternalAggregations; (InternalAggregations.java:187)
at org.elasticsearch.action.search.QueryPhaseResultConsumer.aggregate(Ljava/util/Iterator;Ljava/util/Iterator;ILorg/elasticsearch/search/aggregations/AggregationReduceContext;)Lorg/elasticsearch/search/aggregations/InternalAggregations; (QueryPhaseResultConsumer.java:422)
at org.elasticsearch.action.search.QueryPhaseResultConsumer.reduce()Lorg/elasticsearch/action/search/SearchPhaseController$ReducedQueryPhase; (QueryPhaseResultConsumer.java:254)
at org.elasticsearch.action.search.FetchSearchPhase.innerRun()V (FetchSearchPhase.java:96)
at org.elasticsearch.action.search.FetchSearchPhase$1.doRun()V (FetchSearchPhase.java:83)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run()V (AbstractRunnable.java:27)